From 7c335da35b0ae456294fe69f917ce3261833ba3d Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Fri, 18 Dec 2020 07:20:15 -0500 Subject: [PATCH 1/4] Add support for 2.10 --- .travis.yml | 5 +- development/Dockerfile | 6 +- ...base_configuration.py => configuration.py} | 157 +++++++++++++----- development/dev.env | 35 ++-- development/docker-compose.yml | 6 +- development/netbox_master/configuration.py | 4 - development/netbox_v2.8.3/configuration.py | 4 - .../netbox_v2.9-beta2/configuration.py | 9 - 8 files changed, 144 insertions(+), 82 deletions(-) rename development/{base_configuration.py => configuration.py} (59%) delete mode 100644 development/netbox_master/configuration.py delete mode 100644 development/netbox_v2.8.3/configuration.py delete mode 100644 development/netbox_v2.9-beta2/configuration.py diff --git a/.travis.yml b/.travis.yml index b815518..1e52caa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,9 @@ env: # Each version of NetBox listed here must have a corresponding directory/configuration file # under development/netbox_/configuration.py matrix: - - NETBOX_VER=v2.8.3 - - NETBOX_VER=v2.9-beta2 + - NETBOX_VER=v2.8.9 + - NETBOX_VER=v2.9.9 + - NETBOX_VER=v2.10.1 - NETBOX_VER=master # Encrypted value for PYPI_TOKEN, this secret has been generated with the following command # travis encrypt PYPI_TOKEN= --add env.global --com diff --git a/development/Dockerfile b/development/Dockerfile index c1e555f..af89382 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -2,7 +2,6 @@ ARG python_ver=3.7 FROM python:${python_ver} -ARG netbox_ver=master ENV PYTHONUNBUFFERED 1 RUN mkdir /prom_cache @@ -18,9 +17,9 @@ RUN pip install --upgrade pip\ # ------------------------------------------------------------------------------------- # Remove redis==3.4.1 from the requirements.txt file as a workaround to #4910 # https://github.com/netbox-community/netbox/issues/4910, required for version 2.8.8 and earlier +ARG netbox_ver=master RUN git clone --single-branch --branch ${netbox_ver} https://github.com/netbox-community/netbox.git /opt/netbox/ && \ cd /opt/netbox/ && \ - sed -i '/^redis\=\=/d' /opt/netbox/requirements.txt && \ pip install -r /opt/netbox/requirements.txt # Make the django-debug-toolbar always visible when DEBUG is enabled, @@ -29,9 +28,6 @@ RUN echo "import sys" >> /opt/netbox/netbox/netbox/settings.py && \ echo "TESTING = len(sys.argv) > 1 and sys.argv[1] == 'test'" >> /opt/netbox/netbox/netbox/settings.py && \ echo "DEBUG_TOOLBAR_CONFIG = {'SHOW_TOOLBAR_CALLBACK': lambda _: DEBUG and not TESTING }" >> /opt/netbox/netbox/netbox/settings.py -# Work around https://github.com/rq/django-rq/issues/421 -RUN pip install django-rq==2.3.2 - # ------------------------------------------------------------------------------------- # Install Netbox Plugin # ------------------------------------------------------------------------------------- diff --git a/development/base_configuration.py b/development/configuration.py similarity index 59% rename from development/base_configuration.py rename to development/configuration.py index 8d98197..f34f9c9 100644 --- a/development/base_configuration.py +++ b/development/configuration.py @@ -1,5 +1,39 @@ -"""NetBox configuration file.""" +"""NetBox configuration.""" import os +from distutils.util import strtobool +from django.core.exceptions import ImproperlyConfigured +from .settings import VERSION # pylint: disable=relative-beyond-top-level + + +# Enforce required configuration parameters +for key in [ + "ALLOWED_HOSTS", + "POSTGRES_DB", + "POSTGRES_USER", + "POSTGRES_HOST", + "POSTGRES_PASSWORD", + "REDIS_HOST", + "REDIS_PASSWORD", + "SECRET_KEY", +]: + if not os.environ.get(key): + raise ImproperlyConfigured(f"Required environment variable {key} is missing.") + + +def is_truthy(arg): + """Convert "truthy" strings into Booleans. + + Examples: + >>> is_truthy('yes') + True + Args: + arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no, + f, false, off and 0. Raises ValueError if val is anything else. + """ + if isinstance(arg, bool): + return arg + return bool(strtobool(arg)) + # For reference see http://netbox.readthedocs.io/en/latest/configuration/mandatory-settings/ # Based on https://github.com/digitalocean/netbox/blob/develop/netbox/netbox/configuration.example.py @@ -16,46 +50,52 @@ # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. # # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] -ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(" ") +ALLOWED_HOSTS = os.environ["ALLOWED_HOSTS"].split(" ") # PostgreSQL database configuration. DATABASE = { - "NAME": os.environ.get("DB_NAME", "netbox"), # Database name - "USER": os.environ.get("DB_USER", ""), # PostgreSQL username - "PASSWORD": os.environ.get("DB_PASSWORD", ""), + "NAME": os.environ["POSTGRES_DB"], # Database name + "USER": os.environ["POSTGRES_USER"], # PostgreSQL username + "PASSWORD": os.environ["POSTGRES_PASSWORD"], # PostgreSQL password - "HOST": os.environ.get("DB_HOST", "localhost"), # Database server - "PORT": os.environ.get("DB_PORT", ""), # Database port (leave blank for default) + "HOST": os.environ["POSTGRES_HOST"], # Database server + "PORT": 5432 if not os.environ.get("POSTGRES_PORT", False) else int(os.environ["POSTGRES_PORT"]), # Database port } # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and # symbols. NetBox will not run without this defined. For more information, see # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-SECRET_KEY -SECRET_KEY = os.environ.get("SECRET_KEY", "") +SECRET_KEY = os.environ["SECRET_KEY"] # Redis database settings. The Redis database is used for caching and background processing such as webhooks # Seperate sections for webhooks and caching allow for connecting to seperate Redis instances/datbases if desired. # Full connection details are required in both sections, even if they are the same. REDIS = { "caching": { - "HOST": os.environ.get("REDIS_HOST", "redis"), + "HOST": os.environ["REDIS_HOST"], "PORT": int(os.environ.get("REDIS_PORT", 6379)), - "PASSWORD": os.environ.get("REDIS_PASSWORD", ""), + "PASSWORD": os.environ["REDIS_PASSWORD"], "DATABASE": 1, - "DEFAULT_TIMEOUT": 300, - "SSL": bool(os.environ.get("REDIS_SSL", False)), + "SSL": is_truthy(os.environ.get("REDIS_SSL", False)), }, "tasks": { - "HOST": os.environ.get("REDIS_HOST", "redis"), + "HOST": os.environ["REDIS_HOST"], "PORT": int(os.environ.get("REDIS_PORT", 6379)), - "PASSWORD": os.environ.get("REDIS_PASSWORD", ""), + "PASSWORD": os.environ["REDIS_PASSWORD"], "DATABASE": 0, - "DEFAULT_TIMEOUT": 300, - "SSL": bool(os.environ.get("REDIS_SSL", False)), + "SSL": is_truthy(os.environ.get("REDIS_SSL", False)), }, } +if VERSION.startswith("2.8."): + # NetBox 2.8.x Specific Settings + REDIS["caching"]["DEFAULT_TIMEOUT"] = 300 + REDIS["tasks"]["DEFAULT_TIMEOUT"] = 300 +elif VERSION.startswith("2.9.") or VERSION.startswith("2.10."): + RQ_DEFAULT_TIMEOUT = 300 +else: + raise ImproperlyConfigured(f"Version {VERSION} of NetBox is unsupported at this time.") ######################### # # @@ -71,8 +111,8 @@ # Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same # content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. -BANNER_TOP = os.environ.get("BANNER_TOP", None) -BANNER_BOTTOM = os.environ.get("BANNER_BOTTOM", None) +BANNER_TOP = os.environ.get("BANNER_TOP", "") +BANNER_BOTTOM = os.environ.get("BANNER_BOTTOM", "") # Text to include on the login page above the login form. HTML is allowed. BANNER_LOGIN = os.environ.get("BANNER_LOGIN", "") @@ -87,45 +127,65 @@ # API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be # allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or # CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers -CORS_ORIGIN_ALLOW_ALL = True +CORS_ORIGIN_ALLOW_ALL = is_truthy(os.environ.get("CORS_ORIGIN_ALLOW_ALL", False)) CORS_ORIGIN_WHITELIST = [] CORS_ORIGIN_REGEX_WHITELIST = [] # Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal # sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging # on a production system. -DEBUG = True -DEVELOPER = True +DEBUG = is_truthy(os.environ.get("DEBUG", False)) +DEVELOPER = is_truthy(os.environ.get("DEVELOPER", False)) # Email settings EMAIL = { - "SERVER": "localhost", - "PORT": 25, - "USERNAME": "", - "PASSWORD": "", - "TIMEOUT": 10, - "FROM_EMAIL": "", + "SERVER": os.environ.get("EMAIL_SERVER", "localhost"), + "PORT": int(os.environ.get("EMAIL_PORT", 25)), + "USERNAME": os.environ.get("EMAIL_USERNAME", ""), + "PASSWORD": os.environ.get("EMAIL_PASSWORD", ""), + "TIMEOUT": int(os.environ.get("EMAIL_TIMEOUT", 10)), # seconds + "FROM_EMAIL": os.environ.get("EMAIL_FROM", ""), } # Enforcement of unique IP space can be toggled on a per-VRF basis. # To enforce unique IP space within the global table (all prefixes and IP addresses not assigned to a VRF), # set ENFORCE_GLOBAL_UNIQUE to True. -ENFORCE_GLOBAL_UNIQUE = False +ENFORCE_GLOBAL_UNIQUE = is_truthy(os.environ.get("ENFORCE_GLOBAL_UNIQUE", False)) + +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +# HTTP_PROXIES = { +# "http": "http://192.0.2.1:3128", +# "https": "http://192.0.2.1:1080", +# } + +# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing +# NetBox from an internal IP. +INTERNAL_IPS = ("127.0.0.1", "::1") + +LOG_LEVEL = os.environ.get("LOG_LEVEL", "DEBUG" if DEBUG else "INFO") # Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: # https://docs.djangoproject.com/en/1.11/topics/logging/ -LOGGING = {} +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "{asctime} {levelname} {message} - {name} - {module} - {pathname}:{lineno}", + "datefmt": "%H:%M:%S", + "style": "{", + }, + }, + "handlers": {"console": {"level": "DEBUG", "class": "rq.utils.ColorizingStreamHandler", "formatter": "verbose"}}, + "root": {"handlers": ["console"], "level": LOG_LEVEL}, +} # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. -LOGIN_REQUIRED = False - -# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: -# BASE_PATH = 'netbox/' -BASE_PATH = os.environ.get("BASE_PATH", "") +LOGIN_REQUIRED = is_truthy(os.environ.get("LOGIN_REQUIRED", False)) # Setting this to True will display a "maintenance mode" banner at the top of every page. -MAINTENANCE_MODE = os.environ.get("MAINTENANCE_MODE", False) +MAINTENANCE_MODE = is_truthy(os.environ.get("MAINTENANCE_MODE", False)) # An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. # "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request @@ -136,11 +196,14 @@ # the default value of this setting is derived from the installed location. MEDIA_ROOT = os.environ.get("MEDIA_ROOT", os.path.join(BASE_DIR, "media")) +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +METRICS_ENABLED = True + NAPALM_USERNAME = os.environ.get("NAPALM_USERNAME", "") NAPALM_PASSWORD = os.environ.get("NAPALM_PASSWORD", "") # NAPALM timeout (in seconds). (Default: 30) -NAPALM_TIMEOUT = os.environ.get("NAPALM_TIMEOUT", 30) +NAPALM_TIMEOUT = int(os.environ.get("NAPALM_TIMEOUT", 30)) # NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must # be provided as a dictionary. @@ -150,26 +213,34 @@ } # Determine how many objects to display per page within a list. (Default: 50) -PAGINATE_COUNT = os.environ.get("PAGINATE_COUNT", 50) +PAGINATE_COUNT = int(os.environ.get("PAGINATE_COUNT", 50)) # Enable installed plugins. Add the name of each plugin to the list. PLUGINS = ["netbox_metrics_ext"] # Plugins configuration settings. These settings are used by various plugins that the user may have installed. # Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. -# PLUGINS_CONFIG = {} +PLUGINS_CONFIG = {} # When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to # prefer IPv4 instead. -PREFER_IPV4 = os.environ.get("PREFER_IPV4", False) +PREFER_IPV4 = is_truthy(os.environ.get("PREFER_IPV4", False)) # Remote authentication support REMOTE_AUTH_ENABLED = False -REMOTE_AUTH_BACKEND = "utilities.auth_backends.RemoteUserBackend" REMOTE_AUTH_HEADER = "HTTP_REMOTE_USER" REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] -REMOTE_AUTH_DEFAULT_PERMISSIONS = [] + +if VERSION.startswith("2.8."): + # NetBox 2.8.x Specific Settings + REMOTE_AUTH_BACKEND = "utilities.auth_backends.RemoteUserBackend" + REMOTE_AUTH_DEFAULT_PERMISSIONS = [] +elif VERSION.startswith("2.9.") or VERSION.startswith("2.10."): + REMOTE_AUTH_BACKEND = "netbox.authentication.RemoteUserBackend" + REMOTE_AUTH_DEFAULT_PERMISSIONS = {} +else: + raise ImproperlyConfigured(f"Version {VERSION} of NetBox is unsupported at this time.") # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. RELEASE_CHECK_TIMEOUT = 24 * 3600 @@ -185,6 +256,10 @@ # this setting is derived from the installed location. REPORTS_ROOT = os.environ.get("REPORTS_ROOT", os.path.join(BASE_DIR, "reports")) +# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +SCRIPTS_ROOT = os.environ.get("SCRIPTS_ROOT", os.path.join(BASE_DIR, "scripts")) + # Time zone (default: UTC) TIME_ZONE = os.environ.get("TIME_ZONE", "UTC") diff --git a/development/dev.env b/development/dev.env index 7f813ec..4a404c8 100644 --- a/development/dev.env +++ b/development/dev.env @@ -1,19 +1,28 @@ ALLOWED_HOSTS=* -DB_NAME=netbox -DB_USER=netbox -DB_PASSWORD=decinablesprewad -PGPASSWORD=decinablesprewad -DB_HOST=postgres -NAPALM_TIMEOUT=5 +BANNER_TOP="Metrics Ext plugin dev" +CHANGELOG_RETENTION=0 +DEBUG=True +DEVELOPER=True +EMAIL_FROM=netbox@example.com +EMAIL_PASSWORD= +EMAIL_PORT=25 +EMAIL_SERVER=localhost +EMAIL_TIMEOUT=5 +EMAIL_USERNAME=netbox MAX_PAGE_SIZE=0 -SECRET_KEY=bqn8nn4qmjvx4hv2u5qr4pp46s3w9skbb63y -POSTGRES_USER=netbox -POSTGRES_PASSWORD=decinablesprewad +METRICS_ENABLED=True +NAPALM_TIMEOUT=5 POSTGRES_DB=netbox -CHANGELOG_RETENTION=0 +POSTGRES_HOST=postgres +POSTGRES_PASSWORD=notverysecurepwd +POSTGRES_USER=netbox REDIS_HOST=redis +REDIS_PASSWORD=notverysecurepwd REDIS_PORT=6379 -# Uncomment REDIS_SSL if using SSL # REDIS_SSL=True -REDIS_PASSWORD=decinablesprewad - +# Uncomment REDIS_SSL if using SSL +SECRET_KEY=r8OwDznj!!dci#P9ghmRfdu1Ysxm0AiPeDCQhKE+N_rClfWNj +SUPERUSER_API_TOKEN=0123456789abcdef0123456789abcdef01234567 +SUPERUSER_EMAIL=admin@example.com +SUPERUSER_NAME=admin +SUPERUSER_PASSWORD=admin \ No newline at end of file diff --git a/development/docker-compose.yml b/development/docker-compose.yml index 700f4b5..1899205 100644 --- a/development/docker-compose.yml +++ b/development/docker-compose.yml @@ -17,8 +17,7 @@ services: env_file: - ./dev.env volumes: - - ./base_configuration.py:/opt/netbox/netbox/netbox/base_configuration.py - - ./netbox_${NETBOX_VER}/configuration.py:/opt/netbox/netbox/netbox/configuration.py + - ./configuration.py:/opt/netbox/netbox/netbox/configuration.py - ./example_reports:/opt/netbox/netbox/reports - ../netbox_metrics_ext:/source/netbox_metrics_ext tty: true @@ -33,8 +32,7 @@ services: env_file: - ./dev.env volumes: - - ./base_configuration.py:/opt/netbox/netbox/netbox/base_configuration.py - - ./netbox_${NETBOX_VER}/configuration.py:/opt/netbox/netbox/netbox/configuration.py + - ./configuration.py:/opt/netbox/netbox/netbox/configuration.py - ./example_reports:/opt/netbox/netbox/reports - ../netbox_metrics_ext:/source/netbox_metrics_ext tty: true diff --git a/development/netbox_master/configuration.py b/development/netbox_master/configuration.py deleted file mode 100644 index 81abf04..0000000 --- a/development/netbox_master/configuration.py +++ /dev/null @@ -1,4 +0,0 @@ -"""NetBox configuration file overrides specific to the latest MASTER version.""" -from .base_configuration import * # pylint: disable=relative-beyond-top-level, wildcard-import - -# Overrides specific to this version go here diff --git a/development/netbox_v2.8.3/configuration.py b/development/netbox_v2.8.3/configuration.py deleted file mode 100644 index 00f4829..0000000 --- a/development/netbox_v2.8.3/configuration.py +++ /dev/null @@ -1,4 +0,0 @@ -"""NetBox configuration file overrides specific to version 2.8.3.""" -from .base_configuration import * # pylint: disable=relative-beyond-top-level, wildcard-import - -# Overrides specific to this version go here diff --git a/development/netbox_v2.9-beta2/configuration.py b/development/netbox_v2.9-beta2/configuration.py deleted file mode 100644 index b3d1df2..0000000 --- a/development/netbox_v2.9-beta2/configuration.py +++ /dev/null @@ -1,9 +0,0 @@ -"""NetBox configuration file overrides specific to the latest MASTER version.""" -from .base_configuration import * # pylint: disable=relative-beyond-top-level, wildcard-import - -REMOTE_AUTH_ENABLED = False -REMOTE_AUTH_BACKEND = "netbox.authentication.RemoteUserBackend" -REMOTE_AUTH_HEADER = "HTTP_REMOTE_USER" -REMOTE_AUTH_AUTO_CREATE_USER = True -REMOTE_AUTH_DEFAULT_GROUPS = [] -REMOTE_AUTH_DEFAULT_PERMISSIONS = {} From da5a310e94d248e67e497d13927870c9f5436c74 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Fri, 18 Dec 2020 07:38:43 -0500 Subject: [PATCH 2/4] Remove master from travis --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1e52caa..b9f193d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ env: - NETBOX_VER=v2.8.9 - NETBOX_VER=v2.9.9 - NETBOX_VER=v2.10.1 - - NETBOX_VER=master # Encrypted value for PYPI_TOKEN, this secret has been generated with the following command # travis encrypt PYPI_TOKEN= --add env.global --com global: From 795f3c5f4cc8f43628815e8c8f7484b8df6409a7 Mon Sep 17 00:00:00 2001 From: nniehoff Date: Fri, 5 Feb 2021 14:20:22 -0700 Subject: [PATCH 3/4] Updating for current NetBox versions --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9f193d..bc27fb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,8 +9,8 @@ env: # under development/netbox_/configuration.py matrix: - NETBOX_VER=v2.8.9 - - NETBOX_VER=v2.9.9 - - NETBOX_VER=v2.10.1 + - NETBOX_VER=v2.9.11 + - NETBOX_VER=v2.10.4 # Encrypted value for PYPI_TOKEN, this secret has been generated with the following command # travis encrypt PYPI_TOKEN= --add env.global --com global: From 0cbd15cb67e8ddcdc2385c2b6974c71d3c23fd5b Mon Sep 17 00:00:00 2001 From: nniehoff Date: Fri, 5 Feb 2021 14:31:29 -0700 Subject: [PATCH 4/4] Using NetBox ver variable --- development/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development/Dockerfile b/development/Dockerfile index af89382..767e163 100644 --- a/development/Dockerfile +++ b/development/Dockerfile @@ -17,7 +17,7 @@ RUN pip install --upgrade pip\ # ------------------------------------------------------------------------------------- # Remove redis==3.4.1 from the requirements.txt file as a workaround to #4910 # https://github.com/netbox-community/netbox/issues/4910, required for version 2.8.8 and earlier -ARG netbox_ver=master +ARG netbox_ver=${NETBOX_VER} RUN git clone --single-branch --branch ${netbox_ver} https://github.com/netbox-community/netbox.git /opt/netbox/ && \ cd /opt/netbox/ && \ pip install -r /opt/netbox/requirements.txt