diff --git a/.gitignore b/.gitignore index 69c335d..9e17f72 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.retry *.code-workspace *.sw? +.venv diff --git a/README.adoc b/README.adoc index b46b331..f738bec 100644 --- a/README.adoc +++ b/README.adoc @@ -36,6 +36,7 @@ Provided you have Ansible installed and are using defaults: [source,bash,subs="attributes"] ---- ansible-galaxy install geerlingguy.postgresql davidwittman.redis {role} +ansible-galaxy collection install community.postgresql ansible-playbook -i your.server.fqdn, ~/.ansible/roles/{role}/examples/playbook_single_host_deploy.yml -K ---- @@ -47,6 +48,7 @@ You can also use Vagrant, if you prefer, to bring up NetBox at `localhost:8080`: [source,bash,subs="attributes"] ---- ansible-galaxy install geerlingguy.postgresql davidwittman.redis {role} +ansible-galaxy collection install community.postgresql cd ~/.ansible/roles/{role}/ vagrant up ---- @@ -67,12 +69,23 @@ endif::[] === PostgreSQL -This role does not setup a PostgreSQL server (but will create a database if -needed), so you'll need to setup a PostgreSQL server and create a database user -separate from this role. Take a look at the _Example Playbook_ section. +This role does not setup a PostgreSQL server (but will create a database if needed), so you'll need to setup a PostgreSQL server and create a database user separate from this role. +Take a look at the _Example Playbook_ section. -WARNING: NetBox v2.2.0+ require PostgreSQL 9.4 at the minimum, which may not be -available in your distribution's repos. You may want to use a role for this. +In addition, for Ansible 2.10+, you may need to install the `community.postgresql` collection. +It is recommended to specify this in your playbook's `requirements.yml` file. +For example: + +[source,yaml] +---- +--- +collections: + - name: community.postgresql + version: 3.4.0 +---- + +WARNING: NetBox v2.2.0+ require PostgreSQL 9.4 at the minimum, which may not be available in your distribution's repos. +You may want to use a role for this. === Redis @@ -231,6 +244,15 @@ Use this syntax if your redis is installed with sentinet architecture (multiple the second set of variables if you wish to split your cache database from your webhooks database. +[source,yaml] +---- +netbox_rqworker_processes: 1 +---- + +Specify how many request queue workers should be started by the systemd service. +You can leave this at the default of 1, unless you have a large number of reports, +scripts and other background tasks. + [source,yaml] ---- netbox_config: @@ -422,6 +444,15 @@ netbox_uwsgi_options: {} Specify extra configuration options to insert into `uwsgi.ini` here. This is expected to be a dictionary of key/value pairs, e.g. `buffer-size: 65535`. +[source,yaml] +netbox_uwsgi_in_venv: false + +Toggle `netbox_uwsgi_in_venv` to `true` if you want `uwsgi` to be installed in the same virtual environment as NetBox. +Otherwise, it will be installed system-wide into the library path of the python version used to created the virtual environment (normal/legacy behavior). + +WARNING: There's a possibility that this may become the default in a later version of this role (I think after further cross-platform testing). +See https://github.com/lae/ansible-role-netbox/issues/144[issue #144] for further details. + [source,yaml] netbox_install_epel: true diff --git a/defaults/main.yml b/defaults/main.yml index fbc8286..6e071fb 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -99,4 +99,6 @@ netbox_pip_constraints: # - 'MarkupSafe<2.1.0' netbox_keep_uwsgi_updated: false +netbox_uwsgi_in_venv: false netbox_uwsgi_options: {} +netbox_rqworker_processes: 1 diff --git a/handlers/main.yml b/handlers/main.yml index f12fe30..84debbf 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -19,11 +19,7 @@ - name: restart netbox-rqworker.service systemd: - name: netbox-rqworker.service + name: "netbox-rqworker@{{ item }}.service" state: restarted daemon_reload: true - -- name: reload netbox-rqworker.service - systemd: - name: netbox-rqworker.service - state: reloaded + with_sequence: count="{{ netbox_rqworker_processes }}" diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 994c37c..f93a3f2 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -27,6 +27,7 @@ role_attr_flags: CREATEDB,NOSUPERUSER ## REDIS server install redis_bind: 127.0.0.1 + netbox_rqworker_processes: 2 roles: - geerlingguy.postgresql - davidwittman.redis diff --git a/molecule/default/tests/test_default.py b/molecule/default/tests/test_default.py index 7968eb9..073d89d 100644 --- a/molecule/default/tests/test_default.py +++ b/molecule/default/tests/test_default.py @@ -10,7 +10,8 @@ def test_services(host): services = [ "netbox.socket", "netbox.service", - "netbox-rqworker.service" + "netbox-rqworker@1.service" + "netbox-rqworker@2.service" ] for service in services: s = host.service(service) diff --git a/tasks/deploy_netbox.yml b/tasks/deploy_netbox.yml index 3bc3e98..3c95501 100644 --- a/tasks/deploy_netbox.yml +++ b/tasks/deploy_netbox.yml @@ -80,7 +80,7 @@ py_compile.compile(f, c); os.remove(c)\"" notify: - reload netbox.service - - reload netbox-rqworker.service + - restart netbox-rqworker.service - name: Generate LDAP configuration for NetBox if enabled template: @@ -204,7 +204,7 @@ app_path: "{{ netbox_current_path }}/netbox" virtualenv: "{{ netbox_virtualenv_path }}" - - name: Clear cached data in NetBox + - name: Clear cached data in NetBox < 3 django_manage: command: "invalidate all" app_path: "{{ netbox_current_path }}/netbox" @@ -213,14 +213,16 @@ - netbox_stable and netbox_stable_version is version('3.0.0', '<') or netbox_git and _netbox_git_contains_invalidate_removed.rc != 0 - - name: Clear cached data in NetBox 3.2.3+ + - name: Clear cached data in NetBox 3.2.3 to 3.6 django_manage: command: "clearcache" app_path: "{{ netbox_current_path }}/netbox" virtualenv: "{{ netbox_virtualenv_path }}" when: - netbox_stable and netbox_stable_version is version('3.2.2', '>') - or netbox_git and _netbox_git_contains_add_clearcache.rc == 0 + and netbox_stable_version is version('3.7.0', '<') + or netbox_git and ( _netbox_git_contains_add_clearcache.rc == 0 + and _netbox_git_contains_remove_clearcache.rc != 0 ) become: true become_user: "{{ netbox_user }}" diff --git a/tasks/install_via_git.yml b/tasks/install_via_git.yml index 7e3afda..a1a0615 100644 --- a/tasks/install_via_git.yml +++ b/tasks/install_via_git.yml @@ -15,77 +15,91 @@ group: "{{ netbox_group }}" state: directory -- name: Check existence of commit 1fb67b7, fixing issue netbox#2239 - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^1fb67b791f1a91c624dae4a1cd256e4cf3ddbb77' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_issue_2239_fix - changed_when: false - failed_when: "_netbox_git_contains_issue_2239_fix.rc not in [0, 1]" +- name: Check for presence of various commits in git history + block: + - name: Check existence of commit 1fb67b7, fixing issue netbox#2239 + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^1fb67b791f1a91c624dae4a1cd256e4cf3ddbb77' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_issue_2239_fix + changed_when: false + failed_when: "_netbox_git_contains_issue_2239_fix.rc not in [0, 1]" -- name: Check existence of commit 3590ed3, renaming webhooks to tasks - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^3590ed378d161dd724fad2dc73ff56da746352f8' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_tasks_rename - changed_when: false - failed_when: "_netbox_git_contains_tasks_rename.rc not in [0, 1]" + - name: Check existence of commit 3590ed3, renaming webhooks to tasks + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^3590ed378d161dd724fad2dc73ff56da746352f8' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_tasks_rename + changed_when: false + failed_when: "_netbox_git_contains_tasks_rename.rc not in [0, 1]" -- name: Check existence of commit 90dbe9b, renaming DEFAULT_TIMEOUT to RQ_DEFAULT_TIMEOUT - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^90dbe9bf60ab3c72be10fd070c65c26dca543ca5' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_rq_timeout - changed_when: False - failed_when: "_netbox_git_contains_rq_timeout.rc not in [0, 1]" + - name: Check existence of commit 90dbe9b, renaming DEFAULT_TIMEOUT to RQ_DEFAULT_TIMEOUT + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^90dbe9bf60ab3c72be10fd070c65c26dca543ca5' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_rq_timeout + changed_when: False + failed_when: "_netbox_git_contains_rq_timeout.rc not in [0, 1]" -- name: Check existence of commit f560693, introducing trace_paths management command - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^f560693748d1f35e79ad9d006a8a9b75ef5ae37b' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_trace_paths - changed_when: False - failed_when: "_netbox_git_contains_trace_paths.rc not in [0, 1]" + - name: Check existence of commit f560693, introducing trace_paths management command + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^f560693748d1f35e79ad9d006a8a9b75ef5ae37b' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_trace_paths + changed_when: False + failed_when: "_netbox_git_contains_trace_paths.rc not in [0, 1]" -- name: Check existence of commit e165dca, introducing mkdocs command during update - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^7058d6ca5ae2901383874bcea5fedad31e165dca' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_mkdocs - changed_when: False - failed_when: "_netbox_git_contains_mkdocs.rc not in [0, 1]" + - name: Check existence of commit e165dca, introducing mkdocs command during update + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^7058d6ca5ae2901383874bcea5fedad31e165dca' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_mkdocs + changed_when: False + failed_when: "_netbox_git_contains_mkdocs.rc not in [0, 1]" -- name: Check existence of commit d87ec82, introducing nightly housekeeping command - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^d87ec82fe34d92a84ee6f2a97ba0a87a53eed015' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_housekeeping - changed_when: False - failed_when: "_netbox_git_contains_housekeeping.rc not in [0, 1]" + - name: Check existence of commit d87ec82, introducing nightly housekeeping command + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^d87ec82fe34d92a84ee6f2a97ba0a87a53eed015' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_housekeeping + changed_when: False + failed_when: "_netbox_git_contains_housekeeping.rc not in [0, 1]" -- name: Check existence of commit 028c876, removing the invalidate command - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^028c876bcafbaede2731f191512bbebe3f1b6a9e' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_invalidate_removed - changed_when: False - failed_when: "_netbox_git_contains_invalidate_removed.rc not in [0, 1]" + - name: Check existence of commit 028c876, removing the invalidate command + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^028c876bcafbaede2731f191512bbebe3f1b6a9e' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_invalidate_removed + changed_when: False + failed_when: "_netbox_git_contains_invalidate_removed.rc not in [0, 1]" -- name: Check existence of commit b172ae6, adding the clearcache command - shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^b172ae65d24f6dc161c4b609477a4547bb62a50b' - args: - chdir: "{{ netbox_git_repo_path }}" - executable: /bin/bash - register: _netbox_git_contains_add_clearcache - changed_when: False - failed_when: "_netbox_git_contains_add_clearcache.rc not in [0, 1]" + - name: Check existence of commit b172ae6, adding the clearcache command + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^b172ae65d24f6dc161c4b609477a4547bb62a50b' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_add_clearcache + changed_when: False + failed_when: "_netbox_git_contains_add_clearcache.rc not in [0, 1]" + + - name: Check existence of commit 2d1f882, removing the clearcache command + shell: 'set -o pipefail; git log --format=%H "{{ netbox_git_version }}" | grep ^2d1f88272497ca72d2e1eca8e291c04538c6810e' + args: + chdir: "{{ netbox_git_repo_path }}" + executable: /bin/bash + register: _netbox_git_contains_remove_clearcache + changed_when: False + failed_when: "_netbox_git_contains_remove_clearcache.rc not in [0, 1]" + + become: true + become_user: "{{ netbox_user }}" - name: Archive and extract snapshot of git repository shell: 'set -o pipefail; git archive "{{ netbox_git_version }}" | tar -x -C "{{ netbox_git_deploy_path }}"' diff --git a/tasks/load_variables.yml b/tasks/load_variables.yml index c177431..7093067 100644 --- a/tasks/load_variables.yml +++ b/tasks/load_variables.yml @@ -59,3 +59,8 @@ set_fact: _netbox_global_python: "{{ ansible_python_interpreter }}" when: ansible_python_interpreter is defined + +- name: Set the execution uwsgi command + set_fact: + netbox_uwsgi_cmd: "/usr/bin/env uwsgi" + when: "not netbox_uwsgi_in_venv | bool" diff --git a/tasks/main.yml b/tasks/main.yml index b6a574d..9f52ca3 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -42,7 +42,7 @@ home: "{{ netbox_home }}" - name: Ensure Postgres database exists (via socket) - postgresql_db: + community.postgresql.postgresql_db: name: "{{ netbox_database }}" login_user: "{{ netbox_database_user }}" login_unix_socket: "{{ netbox_database_socket }}" @@ -54,7 +54,7 @@ - netbox_database_host is not defined - name: Ensure Postgres database exists (via TCP) - postgresql_db: + community.postgresql.postgresql_db: name: "{{ netbox_database }}" login_host: "{{ netbox_database_host }}" port: "{{ netbox_database_port }}" @@ -72,6 +72,7 @@ name: uwsgi state: "{{ 'latest' if netbox_keep_uwsgi_updated else 'present' }}" umask: "0022" + virtualenv: "{{ omit if not netbox_uwsgi_in_venv else netbox_virtualenv_path }}" environment: PATH: "/usr/local/bin:{{ _path }}" notify: @@ -106,7 +107,7 @@ - name: Install NetBox-rqworker service unit file template: src: netbox-rqworker.service.j2 - dest: /lib/systemd/system/netbox-rqworker.service + dest: /lib/systemd/system/netbox-rqworker@.service notify: - restart netbox-rqworker.service @@ -121,9 +122,10 @@ - name: Start and enable netbox-rqworker.service systemd: - name: netbox-rqworker.service + name: "netbox-rqworker@{{ item }}.service" state: started enabled: true + with_sequence: count="{{ netbox_rqworker_processes }}" - name: Restore the previous Ansible Python interpreter set_fact: diff --git a/templates/netbox-rqworker.service.j2 b/templates/netbox-rqworker.service.j2 index 7937142..070fb7d 100644 --- a/templates/netbox-rqworker.service.j2 +++ b/templates/netbox-rqworker.service.j2 @@ -1,25 +1,25 @@ {{ ansible_managed | comment }} [Unit] -Description=NetBox RQ-Worker -Documentation=http://netbox.readthedocs.io/en/{{ 'latest' if netbox_git else 'stable' }}/installation/3-http-daemon/#supervisord-installation -After=syslog.target +Description=NetBox Request Queue Worker %i +Documentation=https://docs.netbox.dev/ +After=network-online.target +Wants=network-online.target [Service] +Type=simple + WorkingDirectory={{ netbox_shared_path }} -ExecStart={{ netbox_virtualenv_path }}/bin/python \ - {{ netbox_current_path }}/netbox/manage.py rqworker -ExecReload=/bin/kill -1 $MAINPID -ExecStop=/bin/kill -2 $MAINPID +ExecStart={{ netbox_virtualenv_path }}/bin/python {{ netbox_current_path }}/netbox/manage.py rqworker high default low + StandardOutput=journal StandardError=journal User={{ netbox_user }} Group={{ netbox_group }} Restart=on-failure -#SuccessExitStatus=15 17 29 30 -KillSignal=SIGQUIT -StandardError=syslog +RestartSec=30 + NotifyAccess=all -PrivateTmp=yes +PrivateTmp=true ProtectSystem=full DeviceAllow=/dev/null rw DeviceAllow=/dev/urandom r diff --git a/templates/netbox.service.j2 b/templates/netbox.service.j2 index dfd717e..32e110c 100644 --- a/templates/netbox.service.j2 +++ b/templates/netbox.service.j2 @@ -6,7 +6,7 @@ Requires=netbox.socket After=syslog.target [Service] -ExecStart=/usr/bin/env uwsgi --ini {{ netbox_shared_path }}/uwsgi.ini +ExecStart={{ netbox_uwsgi_cmd }} --ini {{ netbox_shared_path }}/uwsgi.ini ExecReload=/bin/kill -1 $MAINPID ExecStop=/bin/kill -2 $MAINPID StandardInput=socket diff --git a/tests/test.yml b/tests/test.yml index be428f9..0acf9f9 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -13,7 +13,7 @@ shell: cat /srv/netbox/shared/uwsgi.ini changed_when: false - name: NetBox rq-worker service status # noqa 303 305 - shell: "systemctl status netbox-rqworker.service" + shell: "systemctl status netbox-rqworker@1.service" changed_when: false - name: NetBox application log # noqa 305 shell: cat /srv/netbox/shared/application.log diff --git a/vars/debian-12.yml b/vars/debian-12.yml new file mode 100644 index 0000000..65a3d78 --- /dev/null +++ b/vars/debian-12.yml @@ -0,0 +1,21 @@ +--- +_netbox_packages: + - libxml2-dev + - libxslt1-dev + - libffi-dev + - libjpeg-dev + - graphviz + - libpq-dev + - libssl-dev + - systemd-cron +_netbox_python_packages: + - python3.11 + - python3.11-dev + - python3-venv + - python3-pip + - python3-psycopg2 # used by ansible's postgres modules +_netbox_python_binary: /usr/bin/python3 +_netbox_ldap_packages: + - libldap2-dev + - libsasl2-dev +netbox_uwsgi_in_venv: true diff --git a/vars/main.yml b/vars/main.yml index 17c39de..f2d7d7f 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -2,6 +2,7 @@ # vars file for lae.netbox netbox_config_path: "{{ netbox_current_path }}/netbox/netbox" netbox_virtualenv_path: "{{ netbox_current_path }}/venv-py3" +netbox_uwsgi_cmd: "{{ netbox_virtualenv_path }}/bin/uwsgi" _netbox_storages_map: s3boto3: boto3 @@ -43,3 +44,4 @@ netbox_superuser_token: | netbox_python_compat_matrix: - { netbox_version_min: '2.8.0', python_needed: '3.6.0' } - { netbox_version_min: '3.0.0', python_needed: '3.7.0' } + - { netbox_version_min: '3.2.0', python_needed: '3.8.0' }