Merge pull request #493 from netbox-community/develop

Release 1.2.0

.github/workflows/push.yml

@@ -14,6 +14,9 @@ jobs:
     name: Checks syntax of our code
     steps:
     - uses: actions/checkout@v2
+      with:
+        # Full git history is needed to get a proper list of changed files within `super-linter`
+        fetch-depth: 0
     - uses: actions/setup-python@v2
     - name: Lint Code Base
       uses: github/super-linter@v3
@@ -39,7 +42,7 @@ jobs:
         build_cmd:
         - ./build-latest.sh
         - PRERELEASE=true ./build-latest.sh
-        - ./build-next.sh
+        - ./build.sh feature
         - ./build.sh develop
         docker_from:
         - '' # use the default of the build script

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
         build_cmd:
         - ./build-latest.sh
         - PRERELEASE=true ./build-latest.sh
-        - ./build-next.sh
+        - ./build.sh feature
         - ./build.sh develop
       fail-fast: false
     runs-on: ubuntu-latest

.gitignore

@@ -7,6 +7,9 @@ configuration/*
 !configuration/configuration.py
 !configuration/extra.py
 configuration/ldap/*
+!configuration/ldap/extra.py
 !configuration/ldap/ldap_config.py
+!configuration/logging.py
+!configuration/plugins.py
 prometheus.yml
 super-linter.log

README.md

@@ -134,8 +134,7 @@ It runs NetBox's own unit tests and ensures that all initializers work:
 IMAGE=netboxcommunity/netbox:latest ./test.sh
 ```
 
-## About
+## Support
 
-This repository is currently maintained and funded by [nxt][nxt].
+This repository is currently maintained by the community.
-
+Please consider sponsoring the maintainers of this project.
-[nxt]: https://nxt.engineering/en/

VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

build-next.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-# Builds develop, develop-* and master branches of NetBox
-
-echo "▶️ $0 $*"
-
-###
-# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
-# and GITHUB_OAUTH_CLIENT_SECRET
-###
-if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
-  echo "🗝 Performing authenticated Github API calls."
-  GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
-else
-  echo "🕶 Performing unauthenticated Github API calls. This might result in lower Github rate limits!"
-  GITHUB_OAUTH_PARAMS=""
-fi
-
-###
-# Calling Github to get the all branches
-###
-ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}"
-GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
-URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
-
-# Composing the JQ commans to extract the most recent version number
-JQ_NEXT='map(.name) | .[] | scan("^[^v].+") | match("^(develop-).*") | .string'
-
-CURL="curl -sS"
-
-# Querying the Github API to fetch all branches
-NEXT=$($CURL "${URL_RELEASES}" | jq -r "$JQ_NEXT")
-
-if [ -n "$NEXT" ]; then
-  # shellcheck disable=SC2068
-  ./build.sh "${NEXT}" $@
-else
-  echo "No branch matching 'develop-*' found"
-  echo "::set-output name=skipped::true"
-fi

configuration/ldap/extra.py

@@ -0,0 +1,28 @@
+####
+## This file contains extra configuration options that can't be configured
+## directly through environment variables.
+## All vairables set here overwrite any existing found in ldap_config.py
+####
+
+# # This Python script inherits all the imports from ldap_config.py
+# from django_auth_ldap.config import LDAPGroupQuery # Imported since not in ldap_config.py
+
+# # Sets a base requirement of membetship to netbox-user-ro, netbox-user-rw, or netbox-user-admin.
+# AUTH_LDAP_REQUIRE_GROUP = (
+#     LDAPGroupQuery("cn=netbox-user-ro,ou=groups,dc=example,dc=com")
+#     | LDAPGroupQuery("cn=netbox-user-rw,ou=groups,dc=example,dc=com")
+#     | LDAPGroupQuery("cn=netbox-user-admin,ou=groups,dc=example,dc=com")
+# )
+
+# # Sets LDAP Flag groups variables with example.
+# AUTH_LDAP_USER_FLAGS_BY_GROUP = {
+#     "is_staff": (
+#         LDAPGroupQuery("cn=netbox-user-ro,ou=groups,dc=example,dc=com")
+#         | LDAPGroupQuery("cn=netbox-user-rw,ou=groups,dc=example,dc=com")
+#         | LDAPGroupQuery("cn=netbox-user-admin,ou=groups,dc=example,dc=com")
+#     ),
+#     "is_superuser": "cn=netbox-user-admin,ou=groups,dc=example,dc=com",
+# }
+
+# # Sets LDAP Mirror groups variables with example groups
+# AUTH_LDAP_MIRROR_GROUPS = ["netbox-user-ro", "netbox-user-rw", "netbox-user-admin"]

configuration/logging.py

@@ -0,0 +1,55 @@
+# # Remove first comment(#) on each line to implement this working logging example.
+# # Add LOGLEVEL environment variable to netbox if you use this example & want a different log level.
+# from os import environ
+
+# # Set LOGLEVEL in netbox.env or docker-compose.overide.yml to override a logging level of INFO.
+# LOGLEVEL = environ.get('LOGLEVEL', 'INFO')
+
+# LOGGING = {
+
+#    'version': 1,
+#    'disable_existing_loggers': False,
+#    'formatters': {
+#        'verbose': {
+#            'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}',
+#            'style': '{',
+#        },
+#        'simple': {
+#            'format': '{levelname} {message}',
+#            'style': '{',
+#        },
+#    },
+#    'filters': {
+#        'require_debug_false': {
+#            '()': 'django.utils.log.RequireDebugFalse',
+#        },
+#    },
+#    'handlers': {
+#        'console': {
+#            'level': LOGLEVEL,
+#            'filters': ['require_debug_false'],
+#            'class': 'logging.StreamHandler',
+#            'formatter': 'simple'
+#        },
+#        'mail_admins': {
+#            'level': 'ERROR',
+#            'class': 'django.utils.log.AdminEmailHandler',
+#            'filters': ['require_debug_false']
+#        }
+#    },
+#    'loggers': {
+#        'django': {
+#            'handlers': ['console'],
+#            'propagate': True,
+#        },
+#        'django.request': {
+#            'handlers': ['mail_admins'],
+#            'level': 'ERROR',
+#            'propagate': False,
+#        },
+#        'django_auth_ldap': {
+#            'handlers': ['console',],
+#            'level': LOGLEVEL,
+#        }
+#    }
+# }

configuration/plugins.py

@@ -0,0 +1,13 @@
+# Add your plugins and plugin settings here.
+# Of course uncomment this file out.
+
+# To learn how to build images with your required plugins
+# See https://github.com/netbox-community/netbox-docker/wiki/Using-Netbox-Plugins
+
+# PLUGINS = ["netbox_bgp"]
+
+# PLUGINS_CONFIG = {
+#   "netbox_bgp": {
+#     ADD YOUR SETTINGS HERE
+#   }
+# }

docker-compose.override.yml.example

@@ -0,0 +1,5 @@
+version: '3.4'
+services:
+  netbox:
+    ports:
+      - 8000:8080

docker-compose.yml

@@ -16,8 +16,6 @@ services:
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
     - netbox-media-files:/opt/netbox/netbox/media:z
-    ports:
-    - "8080"
   netbox-worker:
     <<: *netbox
     depends_on:

docker/docker-entrypoint.sh

@@ -68,6 +68,4 @@ echo "✅ Initialisation is done."
 
 # Launch whatever is passed by docker
 # (i.e. the RUN instruction in the Dockerfile)
-#
+exec "$@"
-# shellcheck disable=SC2068
-exec $@

initializers/custom_links.yml

@@ -10,12 +10,12 @@
 ## Examples:
 
 # - name: link_to_repo
-#   text: 'Link to Netbox Docker'
+#   link_text: 'Link to Netbox Docker'
-#   url: 'https://github.com/netbox-community/netbox-docker'
+#   link_url: 'https://github.com/netbox-community/netbox-docker'
 #   new_window: False
 #   content_type: device
 # - name: link_to_localhost
-#   text: 'Link to localhost'
+#   link_text: 'Link to localhost'
-#   url: 'http://localhost'
+#   link_url: 'http://localhost'
 #   new_window: True
 #   content_type: device

initializers/devices.yml

@@ -42,3 +42,12 @@
 #   position: 3
 #   custom_field_data:
 #     text_field: Description
+# - name: server04
+#   device_role: server
+#   device_type: Other
+#   site: SING 1
+#   location: cage 101
+#   face: front
+#   position: 3
+#   custom_field_data:
+#     text_field: Description

initializers/groups.yml

@@ -1,15 +1,3 @@
-## To list all permissions, run:
-## 
-## docker-compose run --rm --entrypoint /bin/bash netbox
-## $ ./manage.py migrate
-## $ ./manage.py shell
-## > from django.contrib.auth.models import Permission
-## > print('\n'.join([p.codename for p in Permission.objects.all()]))
-##
-## Permission lists support wildcards. See the examples below.
-##
-## Examples:
-
 # applications:
 #   users:
 #     - technical_user
@@ -19,17 +7,3 @@
 # writers:
 #   users:
 #     - writer
-#   permissions:
-#   - delete_device
-#   - delete_virtualmachine
-#   - add_*
-#   - change_*
-# vm_managers:
-#   permissions:
-#   - '*_virtualmachine'
-# device_managers:
-#   permissions:
-#   - '*device*'
-# creators:
-#   permissions:
-#   - add_*

initializers/object_permissions.yml

@@ -0,0 +1,48 @@
+# all.ro:
+#   actions:
+#     - view
+#   description: 'Read Only for All Objects'
+#   enabled: true
+#   groups:
+#     - applications
+#     - readers
+#   object_types: all
+#   users:
+#     - jdoe
+# all.rw:
+#   actions:
+#     - add
+#     - change
+#     - delete
+#     - view
+#   description: 'Read/Write for All Objects'
+#   enabled: true
+#   groups:
+#     - writers
+#   object_types: all
+# network_team.rw:
+#   actions:
+#     - add
+#     - change
+#     - delete
+#     - view
+#   description: "Network Team Permissions"
+#   enabled: true
+#   object_types:
+#     circuits:
+#       - circuit
+#       - circuittermination
+#       - circuittype
+#       - provider
+#     dcim: all
+#     ipam:
+#       - aggregate
+#       - ipaddress
+#       - prefix
+#       - rir
+#       - role
+#       - routetarget
+#       - service
+#       - vlan
+#       - vlangroup
+#       - vrf

initializers/power_panels.yml

@@ -2,4 +2,4 @@
 #   site: AMS 1   
 # - name: power panel SING 1
 #   site: SING 1 
-#   rack_group: cage 101
+#   location: cage 101

initializers/racks.yml

@@ -32,7 +32,7 @@
 #     text_field: Description
 # - site: SING 1
 #   name: rack-03
-#   group: cage 101
+#   location: cage 101
 #   role: Role 3
 #   type: 4-post-cabinet
 #   width: 19

initializers/users.yml

@@ -1,23 +1,14 @@
-## To list all permissions, run:
-## 
-## docker-compose run --rm --entrypoint /bin/bash netbox
-## $ ./manage.py migrate
-## $ ./manage.py shell
-## > from django.contrib.auth.models import Permission
-## > print('\n'.join([p.codename for p in Permission.objects.all()]))
-##
-## Permission lists support wildcards. See the examples below.
-##
-## Examples:
-
 # technical_user:
 #   api_token: 0123456789technicaluser789abcdef01234567 # must be looooong!
 # reader:
 #   password: reader
 # writer:
 #   password: writer
-#   permissions:
+# jdoe:
-#   - delete_device
+#   first_name: John
-#   - delete_virtualmachine
+#   last_name: Doe
-#   - add_*
+#   api_token: 0123456789jdoe789abcdef01234567jdoe
-#   - change_*
+#   is_active: True
+#   is_superuser: False
+#   is_staff: False
+#   email: john.doe@example.com

initializers/vlan_groups.yml

@@ -1,6 +1,24 @@
 # - name: VLAN group 1
-#   site: AMS 1
+#   scope_type: dcim.region
+#   scope: Amsterdam
 #   slug: vlan-group-1
 # - name: VLAN group 2
-#   site: AMS 1
+#   scope_type: dcim.site
+#   scope: AMS 1
 #   slug: vlan-group-2
+# - name: VLAN group 3
+#   scope_type: dcim.location
+#   scope: cage 101
+#   slug: vlan-group-3
+# - name: VLAN group 4
+#   scope_type: dcim.rack
+#   scope: rack-01
+#   slug: vlan-group-4
+# - name: VLAN group 5
+#   scope_type: virtualization.cluster
+#   scope: cluster1
+#   slug: vlan-group-5
+# - name: VLAN group 6
+#   scope_type: virtualization.clustergroup
+#   scope: Group 1
+#   slug: vlan-group-6

requirements-container.txt

@@ -1,4 +1,4 @@
 napalm==3.2.0
-ruamel.yaml==0.16.13
+ruamel.yaml==0.17.4
-django-auth-ldap==2.3.0
+django-auth-ldap==2.4.0
 django-storages[azure,boto3,dropbox,google,libcloud,sftp]==1.11.1

startup_scripts/000_users.py

@@ -1,7 +1,7 @@
 import sys
 
 from django.contrib.auth.models import User
-from startup_script_utils import load_yaml, set_permissions
+from startup_script_utils import load_yaml
 from users.models import Token
 
 users = load_yaml("/opt/netbox/initializers/users.yml")
@@ -19,6 +19,3 @@ for username, user_details in users.items():
 
         if user_details.get("api_token", 0):
             Token.objects.create(user=user, key=user_details["api_token"])
-
-        yaml_permissions = user_details.get("permissions", [])
-        set_permissions(user.user_permissions, yaml_permissions)

startup_scripts/010_groups.py

@@ -1,23 +1,23 @@
 import sys
 
-from django.contrib.auth.models import Group, User
+from startup_script_utils import load_yaml
-from startup_script_utils import load_yaml, set_permissions
+from users.models import AdminGroup, AdminUser
 
 groups = load_yaml("/opt/netbox/initializers/groups.yml")
 if groups is None:
     sys.exit()
 
 for groupname, group_details in groups.items():
-    group, created = Group.objects.get_or_create(name=groupname)
+    group, created = AdminGroup.objects.get_or_create(name=groupname)
 
     if created:
         print("👥 Created group", groupname)
 
     for username in group_details.get("users", []):
-        user = User.objects.get(username=username)
+        user = AdminUser.objects.get(username=username)
 
         if user:
-            user.groups.add(group)
+            group.user_set.add(user)
+            print(" 👤 Assigned user %s to group %s" % (username, AdminGroup.name))
 
-    yaml_permissions = group_details.get("permissions", [])
+    group.save()
-    set_permissions(group.permissions, yaml_permissions)

startup_scripts/015_object_permissions.py

@@ -0,0 +1,60 @@
+import sys
+
+from django.contrib.contenttypes.models import ContentType
+from startup_script_utils import load_yaml
+from users.models import AdminGroup, AdminUser, ObjectPermission
+
+object_permissions = load_yaml("/opt/netbox/initializers/object_permissions.yml")
+
+if object_permissions is None:
+    sys.exit()
+
+
+for permission_name, permission_details in object_permissions.items():
+
+    object_permission, created = ObjectPermission.objects.get_or_create(
+        name=permission_name,
+        description=permission_details["description"],
+        enabled=permission_details["enabled"],
+        actions=permission_details["actions"],
+    )
+
+    if permission_details.get("object_types", 0):
+        object_types = permission_details["object_types"]
+
+        if object_types == "all":
+            object_permission.object_types.set(ContentType.objects.all())
+
+        else:
+            for app_label, models in object_types.items():
+                if models == "all":
+                    app_models = ContentType.objects.filter(app_label=app_label)
+
+                    for app_model in app_models:
+                        object_permission.object_types.add(app_model.id)
+                else:
+                    # There is
+                    for model in models:
+                        object_permission.object_types.add(
+                            ContentType.objects.get(app_label=app_label, model=model)
+                        )
+
+    print("🔓 Created object permission", object_permission.name)
+
+    if permission_details.get("groups", 0):
+        for groupname in permission_details["groups"]:
+            group = AdminGroup.objects.filter(name=groupname).first()
+
+            if group:
+                object_permission.groups.add(group)
+                print(" 👥 Assigned group %s object permission of %s" % (groupname, groupname))
+
+    if permission_details.get("users", 0):
+        for username in permission_details["users"]:
+            user = AdminUser.objects.filter(username=username).first()
+
+            if user:
+                object_permission.users.add(user)
+                print(" 👤 Assigned user %s object permission of %s" % (username, groupname))
+
+    object_permission.save()

startup_scripts/075_locations.py

@@ -1,9 +1,9 @@
 import sys
 
-from dcim.models import RackGroup, Site
+from dcim.models import Location, Site
 from startup_script_utils import load_yaml
 
-rack_groups = load_yaml("/opt/netbox/initializers/rack_groups.yml")
+rack_groups = load_yaml("/opt/netbox/initializers/locations.yml")
 
 if rack_groups is None:
     sys.exit()
@@ -17,7 +17,7 @@ for params in rack_groups:
         query = {field: params.pop(assoc)}
         params[assoc] = model.objects.get(**query)
 
-    rack_group, created = RackGroup.objects.get_or_create(**params)
+    location, created = Location.objects.get_or_create(**params)
 
     if created:
-        print("🎨 Created rack group", rack_group.name)
+        print("🎨 Created location", location.name)

startup_scripts/080_racks.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import Rack, RackGroup, RackRole, Site
+from dcim.models import Location, Rack, RackRole, Site
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 from tenancy.models import Tenant
 
@@ -14,7 +14,7 @@ required_assocs = {"site": (Site, "name")}
 optional_assocs = {
     "role": (RackRole, "name"),
     "tenant": (Tenant, "name"),
-    "group": (RackGroup, "name"),
+    "location": (Location, "name"),
 }
 
 for params in racks:

startup_scripts/140_devices.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import Device, DeviceRole, DeviceType, Platform, Rack, Site
+from dcim.models import Device, DeviceRole, DeviceType, Location, Platform, Rack, Site
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 from tenancy.models import Tenant
 from virtualization.models import Cluster
@@ -21,6 +21,7 @@ optional_assocs = {
     "platform": (Platform, "name"),
     "rack": (Rack, "name"),
     "cluster": (Cluster, "name"),
+    "location": (Location, "name"),
 }
 
 for params in devices:

startup_scripts/145_devices.py

@@ -1,51 +0,0 @@
-import sys
-
-from dcim.models import Device, DeviceRole, DeviceType, Platform, Rack, Site
-from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
-from tenancy.models import Tenant
-from virtualization.models import Cluster
-
-devices = load_yaml("/opt/netbox/initializers/devices.yml")
-
-if devices is None:
-    sys.exit()
-
-required_assocs = {
-    "device_role": (DeviceRole, "name"),
-    "device_type": (DeviceType, "model"),
-    "site": (Site, "name"),
-}
-
-optional_assocs = {
-    "tenant": (Tenant, "name"),
-    "platform": (Platform, "name"),
-    "rack": (Rack, "name"),
-    "cluster": (Cluster, "name"),
-}
-
-for params in devices:
-    custom_field_data = pop_custom_fields(params)
-
-    # primary ips are handled later in `270_primary_ips.py`
-    params.pop("primary_ip4", None)
-    params.pop("primary_ip6", None)
-
-    for assoc, details in required_assocs.items():
-        model, field = details
-        query = {field: params.pop(assoc)}
-
-        params[assoc] = model.objects.get(**query)
-
-    for assoc, details in optional_assocs.items():
-        if assoc in params:
-            model, field = details
-            query = {field: params.pop(assoc)}
-
-            params[assoc] = model.objects.get(**query)
-
-    device, created = Device.objects.get_or_create(**params)
-
-    if created:
-        set_custom_fields_values(device, custom_field_data)
-
-        print("🖥️  Created device", device.name)

startup_scripts/200_vlan_groups.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import Site
+from django.contrib.contenttypes.models import ContentType
 from ipam.models import VLANGroup
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 
@@ -9,7 +9,7 @@ vlan_groups = load_yaml("/opt/netbox/initializers/vlan_groups.yml")
 if vlan_groups is None:
     sys.exit()
 
-optional_assocs = {"site": (Site, "name")}
+optional_assocs = {"scope": (None, "name")}
 
 for params in vlan_groups:
     custom_field_data = pop_custom_fields(params)
@@ -18,9 +18,20 @@ for params in vlan_groups:
         if assoc in params:
             model, field = details
             query = {field: params.pop(assoc)}
-
+            # Get model from Contenttype
-            params[assoc] = model.objects.get(**query)
+            scope_type = params.pop("scope_type", None)
-
+            if not scope_type:
+                print(f"VLAN Group '{params['name']}': scope_type is missing from VLAN Group")
+                continue
+            app_label, model = str(scope_type).split(".")
+            ct = ContentType.objects.filter(app_label=app_label, model=model).first()
+            if not ct:
+                print(
+                    f"VLAN Group '{params['name']}': ContentType for "
+                    + f"app_label = '{app_label}' and model = '{model}' not found"
+                )
+                continue
+            params["scope_id"] = ct.model_class().objects.get(**query).id
     vlan_group, created = VLANGroup.objects.get_or_create(**params)
 
     if created:

startup_scripts/280_custom_links.py

@@ -23,7 +23,7 @@ for link in custom_links:
     if link["content_type_id"] is None:
         print(
             "⚠️ Unable to create Custom Link '{0}': The content_type '{1}' is unknown".format(
-                link.name, content_type
+                link.get("name"), content_type
             )
         )
         continue

startup_scripts/330_power_panels.py

@@ -1,6 +1,6 @@
 import sys
 
-from dcim.models import PowerPanel, RackGroup, Site
+from dcim.models import Location, PowerPanel, Site
 from startup_script_utils import load_yaml, pop_custom_fields, set_custom_fields_values
 
 power_panels = load_yaml("/opt/netbox/initializers/power_panels.yml")
@@ -10,7 +10,7 @@ if power_panels is None:
 
 required_assocs = {"site": (Site, "name")}
 
-optional_assocs = {"rack_group": (RackGroup, "name")}
+optional_assocs = {"location": (Location, "name")}
 
 for params in power_panels:
     custom_field_data = pop_custom_fields(params)

startup_scripts/startup_script_utils/__init__.py

@@ -1,3 +1,2 @@
 from .custom_fields import pop_custom_fields, set_custom_fields_values
 from .load_yaml import load_yaml
-from .permissions import set_permissions

startup_scripts/startup_script_utils/permissions.py

@@ -1,22 +0,0 @@
-from django.contrib.auth.models import Permission
-
-
-def set_permissions(subject, permission_filters):
-    if subject is None or permission_filters is None:
-        return
-    subject.clear()
-    for permission_filter in permission_filters:
-        if "*" in permission_filter:
-            permission_filter_regex = "^" + permission_filter.replace("*", ".*") + "$"
-            permissions = Permission.objects.filter(codename__iregex=permission_filter_regex)
-            print(
-                "  ⚿ Granting",
-                permissions.count(),
-                "permissions matching '" + permission_filter + "'",
-            )
-        else:
-            permissions = Permission.objects.filter(codename=permission_filter)
-            print("  ⚿ Granting permission", permission_filter)
-
-        for permission in permissions:
-            subject.add(permission)