Merge pull request #18 from networktocode/dga-2.10

Update Travis to run tests on Netbox 2.10 too

.travis.yml

@@ -8,9 +8,9 @@ env:
 # Each version of NetBox listed here must have a corresponding directory/configuration file 
 # under development/netbox_<NETBOX_VER>/configuration.py
   matrix:
-  - NETBOX_VER=v2.8.3
-  - NETBOX_VER=v2.9-beta2
-  - NETBOX_VER=master
+  - NETBOX_VER=v2.8.9
+  - NETBOX_VER=v2.9.11
+  - NETBOX_VER=v2.10.4
 # Encrypted value for PYPI_TOKEN, this secret has been generated with the following command
 #   travis encrypt PYPI_TOKEN=<value> --add env.global --com
   global:

development/Dockerfile

@@ -2,7 +2,6 @@
 ARG python_ver=3.7
 FROM python:${python_ver}
 
-ARG netbox_ver=master
 ENV PYTHONUNBUFFERED 1
 
 RUN mkdir /prom_cache
@@ -18,9 +17,9 @@ RUN pip install --upgrade pip\
 # -------------------------------------------------------------------------------------
 # Remove redis==3.4.1 from the requirements.txt file as a workaround to #4910
 # https://github.com/netbox-community/netbox/issues/4910, required for version 2.8.8 and earlier
+ARG netbox_ver=${NETBOX_VER}
 RUN git clone --single-branch --branch ${netbox_ver} https://github.com/netbox-community/netbox.git /opt/netbox/ && \
     cd /opt/netbox/ && \
-    sed -i '/^redis\=\=/d' /opt/netbox/requirements.txt && \
     pip install -r /opt/netbox/requirements.txt
 
 # Make the django-debug-toolbar always visible when DEBUG is enabled,
@@ -29,9 +28,6 @@ RUN echo "import sys" >> /opt/netbox/netbox/netbox/settings.py && \
     echo "TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test'" >> /opt/netbox/netbox/netbox/settings.py && \
     echo "DEBUG_TOOLBAR_CONFIG = {'SHOW_TOOLBAR_CALLBACK': lambda _: DEBUG and not TESTING }" >> /opt/netbox/netbox/netbox/settings.py
 
-# Work around https://github.com/rq/django-rq/issues/421
-RUN pip install django-rq==2.3.2
-
 # -------------------------------------------------------------------------------------
 # Install Netbox Plugin
 # -------------------------------------------------------------------------------------

development/base_configuration.py → development/configuration.py

@@ -1,5 +1,39 @@
-"""NetBox configuration file."""
+"""NetBox configuration."""
 import os
+from distutils.util import strtobool
+from django.core.exceptions import ImproperlyConfigured
+from .settings import VERSION  # pylint: disable=relative-beyond-top-level
+
+
+# Enforce required configuration parameters
+for key in [
+    "ALLOWED_HOSTS",
+    "POSTGRES_DB",
+    "POSTGRES_USER",
+    "POSTGRES_HOST",
+    "POSTGRES_PASSWORD",
+    "REDIS_HOST",
+    "REDIS_PASSWORD",
+    "SECRET_KEY",
+]:
+    if not os.environ.get(key):
+        raise ImproperlyConfigured(f"Required environment variable {key} is missing.")
+
+
+def is_truthy(arg):
+    """Convert "truthy" strings into Booleans.
+
+    Examples:
+        >>> is_truthy('yes')
+        True
+    Args:
+        arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no,
+        f, false, off and 0. Raises ValueError if val is anything else.
+    """
+    if isinstance(arg, bool):
+        return arg
+    return bool(strtobool(arg))
+
 
 # For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/
 # Based on https://github.com/digitalocean/netbox/blob/develop/netbox/netbox/configuration.example.py
@@ -16,46 +50,52 @@
 # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name.
 #
 # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local']
-ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(" ")
+ALLOWED_HOSTS = os.environ["ALLOWED_HOSTS"].split(" ")
 
 # PostgreSQL database configuration.
 DATABASE = {
-    "NAME": os.environ.get("DB_NAME", "netbox"),  # Database name
-    "USER": os.environ.get("DB_USER", ""),  # PostgreSQL username
-    "PASSWORD": os.environ.get("DB_PASSWORD", ""),
+    "NAME": os.environ["POSTGRES_DB"],  # Database name
+    "USER": os.environ["POSTGRES_USER"],  # PostgreSQL username
+    "PASSWORD": os.environ["POSTGRES_PASSWORD"],
     # PostgreSQL password
-    "HOST": os.environ.get("DB_HOST", "localhost"),  # Database server
-    "PORT": os.environ.get("DB_PORT", ""),  # Database port (leave blank for default)
+    "HOST": os.environ["POSTGRES_HOST"],  # Database server
+    "PORT": 5432 if not os.environ.get("POSTGRES_PORT", False) else int(os.environ["POSTGRES_PORT"]),  # Database port
 }
 
 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file.
 # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and
 # symbols. NetBox will not run without this defined. For more information, see
 # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY
-SECRET_KEY = os.environ.get("SECRET_KEY", "")
+SECRET_KEY = os.environ["SECRET_KEY"]
 
 # Redis database settings. The Redis database is used for caching and background processing such as webhooks
 # Seperate sections for webhooks and caching allow for connecting to seperate Redis instances/datbases if desired.
 # Full connection details are required in both sections, even if they are the same.
 REDIS = {
     "caching": {
-        "HOST": os.environ.get("REDIS_HOST", "redis"),
+        "HOST": os.environ["REDIS_HOST"],
         "PORT": int(os.environ.get("REDIS_PORT", 6379)),
-        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
+        "PASSWORD": os.environ["REDIS_PASSWORD"],
         "DATABASE": 1,
-        "DEFAULT_TIMEOUT": 300,
-        "SSL": bool(os.environ.get("REDIS_SSL", False)),
+        "SSL": is_truthy(os.environ.get("REDIS_SSL", False)),
     },
     "tasks": {
-        "HOST": os.environ.get("REDIS_HOST", "redis"),
+        "HOST": os.environ["REDIS_HOST"],
         "PORT": int(os.environ.get("REDIS_PORT", 6379)),
-        "PASSWORD": os.environ.get("REDIS_PASSWORD", ""),
+        "PASSWORD": os.environ["REDIS_PASSWORD"],
         "DATABASE": 0,
-        "DEFAULT_TIMEOUT": 300,
-        "SSL": bool(os.environ.get("REDIS_SSL", False)),
+        "SSL": is_truthy(os.environ.get("REDIS_SSL", False)),
     },
 }
 
+if VERSION.startswith("2.8."):
+    # NetBox 2.8.x Specific Settings
+    REDIS["caching"]["DEFAULT_TIMEOUT"] = 300
+    REDIS["tasks"]["DEFAULT_TIMEOUT"] = 300
+elif VERSION.startswith("2.9.") or VERSION.startswith("2.10."):
+    RQ_DEFAULT_TIMEOUT = 300
+else:
+    raise ImproperlyConfigured(f"Version {VERSION} of NetBox is unsupported at this time.")
 
 #########################
 #                       #
@@ -71,8 +111,8 @@
 
 # Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same
 # content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP.
-BANNER_TOP = os.environ.get("BANNER_TOP", None)
-BANNER_BOTTOM = os.environ.get("BANNER_BOTTOM", None)
+BANNER_TOP = os.environ.get("BANNER_TOP", "")
+BANNER_BOTTOM = os.environ.get("BANNER_BOTTOM", "")
 
 # Text to include on the login page above the login form. HTML is allowed.
 BANNER_LOGIN = os.environ.get("BANNER_LOGIN", "")
@@ -87,45 +127,65 @@
 # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be
 # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or
 # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers
-CORS_ORIGIN_ALLOW_ALL = True
+CORS_ORIGIN_ALLOW_ALL = is_truthy(os.environ.get("CORS_ORIGIN_ALLOW_ALL", False))
 CORS_ORIGIN_WHITELIST = []
 CORS_ORIGIN_REGEX_WHITELIST = []
 
 # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal
 # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging
 # on a production system.
-DEBUG = True
-DEVELOPER = True
+DEBUG = is_truthy(os.environ.get("DEBUG", False))
+DEVELOPER = is_truthy(os.environ.get("DEVELOPER", False))
 
 # Email settings
 EMAIL = {
-    "SERVER": "localhost",
-    "PORT": 25,
-    "USERNAME": "",
-    "PASSWORD": "",
-    "TIMEOUT": 10,
-    "FROM_EMAIL": "",
+    "SERVER": os.environ.get("EMAIL_SERVER", "localhost"),
+    "PORT": int(os.environ.get("EMAIL_PORT", 25)),
+    "USERNAME": os.environ.get("EMAIL_USERNAME", ""),
+    "PASSWORD": os.environ.get("EMAIL_PASSWORD", ""),
+    "TIMEOUT": int(os.environ.get("EMAIL_TIMEOUT", 10)),  # seconds
+    "FROM_EMAIL": os.environ.get("EMAIL_FROM", ""),
 }
 
 # Enforcement of unique IP space can be toggled on a per-VRF basis.
 # To enforce unique IP space within the global table (all prefixes and IP addresses not assigned to a VRF),
 # set ENFORCE_GLOBAL_UNIQUE to True.
-ENFORCE_GLOBAL_UNIQUE = False
+ENFORCE_GLOBAL_UNIQUE = is_truthy(os.environ.get("ENFORCE_GLOBAL_UNIQUE", False))
+
+# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks).
+# HTTP_PROXIES = {
+#     "http": "http://192.0.2.1:3128",
+#     "https": "http://192.0.2.1:1080",
+# }
+
+# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing
+# NetBox from an internal IP.
+INTERNAL_IPS = ("127.0.0.1", "::1")
+
+LOG_LEVEL = os.environ.get("LOG_LEVEL", "DEBUG" if DEBUG else "INFO")
 
 # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs:
 #   https://docs.djangoproject.com/en/1.11/topics/logging/
-LOGGING = {}
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "verbose": {
+            "format": "{asctime} {levelname} {message} - {name} - {module} - {pathname}:{lineno}",
+            "datefmt": "%H:%M:%S",
+            "style": "{",
+        },
+    },
+    "handlers": {"console": {"level": "DEBUG", "class": "rq.utils.ColorizingStreamHandler", "formatter": "verbose"}},
+    "root": {"handlers": ["console"], "level": LOG_LEVEL},
+}
 
 # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users
 # are permitted to access most data in NetBox (excluding secrets) but not make any changes.
-LOGIN_REQUIRED = False
-
-# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set:
-# BASE_PATH = 'netbox/'
-BASE_PATH = os.environ.get("BASE_PATH", "")
+LOGIN_REQUIRED = is_truthy(os.environ.get("LOGIN_REQUIRED", False))
 
 # Setting this to True will display a "maintenance mode" banner at the top of every page.
-MAINTENANCE_MODE = os.environ.get("MAINTENANCE_MODE", False)
+MAINTENANCE_MODE = is_truthy(os.environ.get("MAINTENANCE_MODE", False))
 
 # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g.
 # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request
@@ -136,11 +196,14 @@
 # the default value of this setting is derived from the installed location.
 MEDIA_ROOT = os.environ.get("MEDIA_ROOT", os.path.join(BASE_DIR, "media"))
 
+# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics'
+METRICS_ENABLED = True
+
 NAPALM_USERNAME = os.environ.get("NAPALM_USERNAME", "")
 NAPALM_PASSWORD = os.environ.get("NAPALM_PASSWORD", "")
 
 # NAPALM timeout (in seconds). (Default: 30)
-NAPALM_TIMEOUT = os.environ.get("NAPALM_TIMEOUT", 30)
+NAPALM_TIMEOUT = int(os.environ.get("NAPALM_TIMEOUT", 30))
 
 # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must
 # be provided as a dictionary.
@@ -150,26 +213,34 @@
 }
 
 # Determine how many objects to display per page within a list. (Default: 50)
-PAGINATE_COUNT = os.environ.get("PAGINATE_COUNT", 50)
+PAGINATE_COUNT = int(os.environ.get("PAGINATE_COUNT", 50))
 
 # Enable installed plugins. Add the name of each plugin to the list.
 PLUGINS = ["netbox_metrics_ext"]
 
 # Plugins configuration settings. These settings are used by various plugins that the user may have installed.
 # Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings.
-# PLUGINS_CONFIG = {}
+PLUGINS_CONFIG = {}
 
 # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to
 # prefer IPv4 instead.
-PREFER_IPV4 = os.environ.get("PREFER_IPV4", False)
+PREFER_IPV4 = is_truthy(os.environ.get("PREFER_IPV4", False))
 
 # Remote authentication support
 REMOTE_AUTH_ENABLED = False
-REMOTE_AUTH_BACKEND = "utilities.auth_backends.RemoteUserBackend"
 REMOTE_AUTH_HEADER = "HTTP_REMOTE_USER"
 REMOTE_AUTH_AUTO_CREATE_USER = True
 REMOTE_AUTH_DEFAULT_GROUPS = []
-REMOTE_AUTH_DEFAULT_PERMISSIONS = []
+
+if VERSION.startswith("2.8."):
+    # NetBox 2.8.x Specific Settings
+    REMOTE_AUTH_BACKEND = "utilities.auth_backends.RemoteUserBackend"
+    REMOTE_AUTH_DEFAULT_PERMISSIONS = []
+elif VERSION.startswith("2.9.") or VERSION.startswith("2.10."):
+    REMOTE_AUTH_BACKEND = "netbox.authentication.RemoteUserBackend"
+    REMOTE_AUTH_DEFAULT_PERMISSIONS = {}
+else:
+    raise ImproperlyConfigured(f"Version {VERSION} of NetBox is unsupported at this time.")
 
 # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour.
 RELEASE_CHECK_TIMEOUT = 24 * 3600
@@ -185,6 +256,10 @@
 # this setting is derived from the installed location.
 REPORTS_ROOT = os.environ.get("REPORTS_ROOT", os.path.join(BASE_DIR, "reports"))
 
+# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of
+# this setting is derived from the installed location.
+SCRIPTS_ROOT = os.environ.get("SCRIPTS_ROOT", os.path.join(BASE_DIR, "scripts"))
+
 # Time zone (default: UTC)
 TIME_ZONE = os.environ.get("TIME_ZONE", "UTC")
 

development/dev.env

@@ -1,19 +1,28 @@
 ALLOWED_HOSTS=*
-DB_NAME=netbox
-DB_USER=netbox
-DB_PASSWORD=decinablesprewad
-PGPASSWORD=decinablesprewad
-DB_HOST=postgres
-NAPALM_TIMEOUT=5
+BANNER_TOP="Metrics Ext plugin dev"
+CHANGELOG_RETENTION=0
+DEBUG=True
+DEVELOPER=True
+EMAIL_FROM=[email protected]
+EMAIL_PASSWORD=
+EMAIL_PORT=25
+EMAIL_SERVER=localhost
+EMAIL_TIMEOUT=5
+EMAIL_USERNAME=netbox
 MAX_PAGE_SIZE=0
-SECRET_KEY=bqn8nn4qmjvx4hv2u5qr4pp46s3w9skbb63y
-POSTGRES_USER=netbox
-POSTGRES_PASSWORD=decinablesprewad
+METRICS_ENABLED=True
+NAPALM_TIMEOUT=5
 POSTGRES_DB=netbox
-CHANGELOG_RETENTION=0
+POSTGRES_HOST=postgres
+POSTGRES_PASSWORD=notverysecurepwd
+POSTGRES_USER=netbox
 REDIS_HOST=redis
+REDIS_PASSWORD=notverysecurepwd
 REDIS_PORT=6379
-# Uncomment REDIS_SSL if using SSL
 # REDIS_SSL=True
-REDIS_PASSWORD=decinablesprewad
-
+# Uncomment REDIS_SSL if using SSL
+SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj
+SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567
+SUPERUSER_EMAIL=[email protected]
+SUPERUSER_NAME=admin
+SUPERUSER_PASSWORD=admin

development/docker-compose.yml

@@ -17,8 +17,7 @@ services:
     env_file:
       - ./dev.env
     volumes:
-      - ./base_configuration.py:/opt/netbox/netbox/netbox/base_configuration.py
-      - ./netbox_${NETBOX_VER}/configuration.py:/opt/netbox/netbox/netbox/configuration.py
+      - ./configuration.py:/opt/netbox/netbox/netbox/configuration.py
       - ./example_reports:/opt/netbox/netbox/reports
       - ../netbox_metrics_ext:/source/netbox_metrics_ext
     tty: true
@@ -33,8 +32,7 @@ services:
     env_file:
       - ./dev.env
     volumes:
-      - ./base_configuration.py:/opt/netbox/netbox/netbox/base_configuration.py
-      - ./netbox_${NETBOX_VER}/configuration.py:/opt/netbox/netbox/netbox/configuration.py
+      - ./configuration.py:/opt/netbox/netbox/netbox/configuration.py
       - ./example_reports:/opt/netbox/netbox/reports
       - ../netbox_metrics_ext:/source/netbox_metrics_ext
     tty: true