From ef1bca61543a9c56cf08d85a49386e7bcd8dc5bc Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Wed, 9 Aug 2023 15:19:51 +0200 Subject: [PATCH 01/15] Move scripts in bin/* to python/nav/bin/ --- pyproject.toml | 36 +++++++++++++++++++ python/nav/bin/__init__.py | 0 {bin => python/nav/bin}/alertengine.py | 0 {bin => python/nav/bin}/autoenable.py | 0 {bin => python/nav/bin}/collect_active_ip.py | 0 {bin => python/nav/bin}/emailreports.py | 0 .../nav/bin/eventengine.py | 0 .../nav/bin/ipdevpolld.py | 0 {bin => python/nav/bin}/logengine.py | 0 {bin => python/nav/bin}/macwatch.py | 0 {bin => python/nav/bin}/mailin.py | 0 {bin => python/nav/bin}/maintengine.py | 0 bin/nav => python/nav/bin/nav.py | 0 .../nav/bin/navcheckservice.py | 0 bin/navclean => python/nav/bin/navclean.py | 0 bin/navdf => python/nav/bin/navdf.py | 0 bin/navdump => python/nav/bin/navdump.py | 0 bin/naventity => python/nav/bin/naventity.py | 0 .../nav/bin/navoidverify.py | 0 bin/navpgdump => python/nav/bin/navpgdump.py | 0 bin/navsnmp => python/nav/bin/navsnmp.py | 0 bin/navstats => python/nav/bin/navstats.py | 0 bin/navsyncdb => python/nav/bin/navsyncdb.py | 0 .../nav/bin/navsynctypes.py | 0 .../nav/bin/navtopology.py | 0 bin/navuser => python/nav/bin/navuser.py | 0 {bin => python/nav/bin}/netbiostracker.py | 0 {bin => python/nav/bin}/pping.py | 0 {bin => python/nav/bin}/radiusparser.py | 0 {bin => python/nav/bin}/servicemon.py | 0 {bin => python/nav/bin}/smsd.py | 0 {bin => python/nav/bin}/snmptrapd.py | 0 {bin => python/nav/bin}/sortedstats_cacher.py | 0 {bin => python/nav/bin}/start_arnold.py | 0 {bin => python/nav/bin}/t1000.py | 0 .../nav/bin/thresholdmon.py | 0 36 files changed, 36 insertions(+) create mode 100644 python/nav/bin/__init__.py rename {bin => python/nav/bin}/alertengine.py (100%) rename {bin => python/nav/bin}/autoenable.py (100%) rename {bin => python/nav/bin}/collect_active_ip.py (100%) rename {bin => python/nav/bin}/emailreports.py (100%) rename bin/eventengine => python/nav/bin/eventengine.py (100%) rename bin/ipdevpolld => python/nav/bin/ipdevpolld.py (100%) rename {bin => python/nav/bin}/logengine.py (100%) rename {bin => python/nav/bin}/macwatch.py (100%) rename {bin => python/nav/bin}/mailin.py (100%) rename {bin => python/nav/bin}/maintengine.py (100%) rename bin/nav => python/nav/bin/nav.py (100%) rename bin/navcheckservice => python/nav/bin/navcheckservice.py (100%) rename bin/navclean => python/nav/bin/navclean.py (100%) rename bin/navdf => python/nav/bin/navdf.py (100%) rename bin/navdump => python/nav/bin/navdump.py (100%) rename bin/naventity => python/nav/bin/naventity.py (100%) rename bin/navoidverify => python/nav/bin/navoidverify.py (100%) rename bin/navpgdump => python/nav/bin/navpgdump.py (100%) rename bin/navsnmp => python/nav/bin/navsnmp.py (100%) rename bin/navstats => python/nav/bin/navstats.py (100%) rename bin/navsyncdb => python/nav/bin/navsyncdb.py (100%) rename bin/navsynctypes => python/nav/bin/navsynctypes.py (100%) rename bin/navtopology => python/nav/bin/navtopology.py (100%) rename bin/navuser => python/nav/bin/navuser.py (100%) rename {bin => python/nav/bin}/netbiostracker.py (100%) rename {bin => python/nav/bin}/pping.py (100%) rename {bin => python/nav/bin}/radiusparser.py (100%) rename {bin => python/nav/bin}/servicemon.py (100%) rename {bin => python/nav/bin}/smsd.py (100%) rename {bin => python/nav/bin}/snmptrapd.py (100%) rename {bin => python/nav/bin}/sortedstats_cacher.py (100%) rename {bin => python/nav/bin}/start_arnold.py (100%) rename {bin => python/nav/bin}/t1000.py (100%) rename bin/thresholdmon => python/nav/bin/thresholdmon.py (100%) diff --git a/pyproject.toml b/pyproject.toml index c949e0af47..fdd4c7ea39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,42 @@ dependencies = [ [project.urls] Homepage = "https://nav.uninett.no/" +[project.scripts] +alertengine = "nav.bin.alertengine:main" +autoenable = "nav.bin.autoenable:main" +collect_active_ip = "nav.bin.collect_active_ip:main" +emailreports = "nav.bin.emailreports:main" +eventengine = "nav.bin.eventengine:main" +ipdevpolld = "nav.bin.ipdevpolld:main" +logengine = "nav.bin.logengine:main" +macwatch = "nav.bin.macwatch:main" +mailin = "nav.bin.mailin:main" +maintengine = "nav.bin.maintengine:main" +nav = "nav.bin.nav:main" +navcheckservice = "nav.bin.navcheckservice:main" +navclean = "nav.bin.navclean:main" +navdf = "nav.bin.navdf:main" +navdump = "nav.bin.navdump:main" +naventity = "nav.bin.naventity:main" +navoidverify = "nav.bin.navoidverify:main" +navpgdump = "nav.bin.navpgdump:main" +navsnmp = "nav.bin.navsnmp:main" +navstats = "nav.bin.navstats:main" +navsyncdb = "nav.bin.navsyncdb:main" +navsynctypes = "nav.bin.navsynctypes:main" +navtopology = "nav.bin.navtopology:main" +navuser = "nav.bin.navuser:main" +netbiostracker = "nav.bin.netbiostracker:main" +pping = "nav.bin.pping:main" +radiusparser = "nav.bin.radiusparser:main" +servicemon = "nav.bin.servicemon:main" +smsd = "nav.bin.smsd:main" +snmptrapd = "nav.bin.snmptrapd:main" +sortedstats_cacher = "nav.bin.sortedstats_cacher:main" +start_arnold = "nav.bin.start_arnold:main" +t1000 = "nav.bin.t1000:main" +thresholdmon = "nav.bin.thresholdmon:main" + [tool.setuptools] include-package-data = true zip-safe = false diff --git a/python/nav/bin/__init__.py b/python/nav/bin/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/bin/alertengine.py b/python/nav/bin/alertengine.py similarity index 100% rename from bin/alertengine.py rename to python/nav/bin/alertengine.py diff --git a/bin/autoenable.py b/python/nav/bin/autoenable.py similarity index 100% rename from bin/autoenable.py rename to python/nav/bin/autoenable.py diff --git a/bin/collect_active_ip.py b/python/nav/bin/collect_active_ip.py similarity index 100% rename from bin/collect_active_ip.py rename to python/nav/bin/collect_active_ip.py diff --git a/bin/emailreports.py b/python/nav/bin/emailreports.py similarity index 100% rename from bin/emailreports.py rename to python/nav/bin/emailreports.py diff --git a/bin/eventengine b/python/nav/bin/eventengine.py similarity index 100% rename from bin/eventengine rename to python/nav/bin/eventengine.py diff --git a/bin/ipdevpolld b/python/nav/bin/ipdevpolld.py similarity index 100% rename from bin/ipdevpolld rename to python/nav/bin/ipdevpolld.py diff --git a/bin/logengine.py b/python/nav/bin/logengine.py similarity index 100% rename from bin/logengine.py rename to python/nav/bin/logengine.py diff --git a/bin/macwatch.py b/python/nav/bin/macwatch.py similarity index 100% rename from bin/macwatch.py rename to python/nav/bin/macwatch.py diff --git a/bin/mailin.py b/python/nav/bin/mailin.py similarity index 100% rename from bin/mailin.py rename to python/nav/bin/mailin.py diff --git a/bin/maintengine.py b/python/nav/bin/maintengine.py similarity index 100% rename from bin/maintengine.py rename to python/nav/bin/maintengine.py diff --git a/bin/nav b/python/nav/bin/nav.py similarity index 100% rename from bin/nav rename to python/nav/bin/nav.py diff --git a/bin/navcheckservice b/python/nav/bin/navcheckservice.py similarity index 100% rename from bin/navcheckservice rename to python/nav/bin/navcheckservice.py diff --git a/bin/navclean b/python/nav/bin/navclean.py similarity index 100% rename from bin/navclean rename to python/nav/bin/navclean.py diff --git a/bin/navdf b/python/nav/bin/navdf.py similarity index 100% rename from bin/navdf rename to python/nav/bin/navdf.py diff --git a/bin/navdump b/python/nav/bin/navdump.py similarity index 100% rename from bin/navdump rename to python/nav/bin/navdump.py diff --git a/bin/naventity b/python/nav/bin/naventity.py similarity index 100% rename from bin/naventity rename to python/nav/bin/naventity.py diff --git a/bin/navoidverify b/python/nav/bin/navoidverify.py similarity index 100% rename from bin/navoidverify rename to python/nav/bin/navoidverify.py diff --git a/bin/navpgdump b/python/nav/bin/navpgdump.py similarity index 100% rename from bin/navpgdump rename to python/nav/bin/navpgdump.py diff --git a/bin/navsnmp b/python/nav/bin/navsnmp.py similarity index 100% rename from bin/navsnmp rename to python/nav/bin/navsnmp.py diff --git a/bin/navstats b/python/nav/bin/navstats.py similarity index 100% rename from bin/navstats rename to python/nav/bin/navstats.py diff --git a/bin/navsyncdb b/python/nav/bin/navsyncdb.py similarity index 100% rename from bin/navsyncdb rename to python/nav/bin/navsyncdb.py diff --git a/bin/navsynctypes b/python/nav/bin/navsynctypes.py similarity index 100% rename from bin/navsynctypes rename to python/nav/bin/navsynctypes.py diff --git a/bin/navtopology b/python/nav/bin/navtopology.py similarity index 100% rename from bin/navtopology rename to python/nav/bin/navtopology.py diff --git a/bin/navuser b/python/nav/bin/navuser.py similarity index 100% rename from bin/navuser rename to python/nav/bin/navuser.py diff --git a/bin/netbiostracker.py b/python/nav/bin/netbiostracker.py similarity index 100% rename from bin/netbiostracker.py rename to python/nav/bin/netbiostracker.py diff --git a/bin/pping.py b/python/nav/bin/pping.py similarity index 100% rename from bin/pping.py rename to python/nav/bin/pping.py diff --git a/bin/radiusparser.py b/python/nav/bin/radiusparser.py similarity index 100% rename from bin/radiusparser.py rename to python/nav/bin/radiusparser.py diff --git a/bin/servicemon.py b/python/nav/bin/servicemon.py similarity index 100% rename from bin/servicemon.py rename to python/nav/bin/servicemon.py diff --git a/bin/smsd.py b/python/nav/bin/smsd.py similarity index 100% rename from bin/smsd.py rename to python/nav/bin/smsd.py diff --git a/bin/snmptrapd.py b/python/nav/bin/snmptrapd.py similarity index 100% rename from bin/snmptrapd.py rename to python/nav/bin/snmptrapd.py diff --git a/bin/sortedstats_cacher.py b/python/nav/bin/sortedstats_cacher.py similarity index 100% rename from bin/sortedstats_cacher.py rename to python/nav/bin/sortedstats_cacher.py diff --git a/bin/start_arnold.py b/python/nav/bin/start_arnold.py similarity index 100% rename from bin/start_arnold.py rename to python/nav/bin/start_arnold.py diff --git a/bin/t1000.py b/python/nav/bin/t1000.py similarity index 100% rename from bin/t1000.py rename to python/nav/bin/t1000.py diff --git a/bin/thresholdmon b/python/nav/bin/thresholdmon.py similarity index 100% rename from bin/thresholdmon rename to python/nav/bin/thresholdmon.py From ccfd242c1c35702467d7d651f4242fed055d200b Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Fri, 8 Sep 2023 09:21:56 +0200 Subject: [PATCH 02/15] Make main() consistent, update path in tests Not all scripts had a sufficiently pure main()-function. Also, remove bindir-magic from setup.py. --- python/nav/bin/collect_active_ip.py | 18 ++++++++++++------ python/nav/bin/emailreports.py | 6 ++++-- python/nav/bin/nav.py | 13 +++++++------ python/nav/bin/radiusparser.py | 18 ++++++++++-------- python/nav/bin/servicemon.py | 10 ++++++---- python/nav/bin/sortedstats_cacher.py | 10 +++++++--- python/nav/bin/start_arnold.py | 8 +++++--- setup.py | 8 -------- tests/integration/bin_test.py | 7 +++++-- tests/integration/conftest.py | 3 ++- 10 files changed, 58 insertions(+), 43 deletions(-) diff --git a/python/nav/bin/collect_active_ip.py b/python/nav/bin/collect_active_ip.py index bb5cffb55c..625e9a3085 100755 --- a/python/nav/bin/collect_active_ip.py +++ b/python/nav/bin/collect_active_ip.py @@ -37,8 +37,11 @@ _logger = logging.getLogger('nav.ipcollector') -def main(days=None): +def main(args=None): """Controller""" + if args == None: + args = get_parser().parse_args() + days = args.days or None exit_if_already_running() init_generic_logging(logfile=LOGFILE, stderr=False) run(days) @@ -62,9 +65,9 @@ def run(days): _logger.info('Done in %.2f seconds', time.time() - starttime) -if __name__ == '__main__': - _parser = argparse.ArgumentParser() - _parser.add_argument( +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( "-d", "--days", default=None, @@ -72,5 +75,8 @@ def run(days): help="days back in time to start collecting from", ) - _args = _parser.parse_args() - main(_args.days) + return parser + + +if __name__ == '__main__': + main() diff --git a/python/nav/bin/emailreports.py b/python/nav/bin/emailreports.py index b546b092c4..9734f0d2a2 100755 --- a/python/nav/bin/emailreports.py +++ b/python/nav/bin/emailreports.py @@ -33,8 +33,10 @@ _logger = logging.getLogger('emailreports') -def main(args): +def main(args=None): """Send all reports""" + if args == None: + args = get_parser().parse_args() init_generic_logging(logfile=LOGFILE, stderr=False) send_reports(args.period) @@ -50,4 +52,4 @@ def get_parser(): if __name__ == '__main__': - main(get_parser().parse_args()) + main() diff --git a/python/nav/bin/nav.py b/python/nav/bin/nav.py index 11f67f83f4..e032ba293b 100755 --- a/python/nav/bin/nav.py +++ b/python/nav/bin/nav.py @@ -22,10 +22,10 @@ import os.path import argparse import textwrap -from nav import colors +from .. import colors try: - from nav.startstop import ServiceRegistry, CommandFailedError, CrontabError + from ..startstop import ServiceRegistry, CommandFailedError, CrontabError except ImportError: print( "Fatal error: Could not find the nav.startstop module.\nIs your " @@ -45,10 +45,11 @@ sys.exit(1) -def main(args): +def main(args=None): """Main execution point""" - parser = make_argparser() - args = parser.parse_args() + if args == None: + parser = make_argparser() + args = parser.parse_args() try: args.func(args) except AttributeError: @@ -330,4 +331,4 @@ def command_config_install(args): # begin here # ############## if __name__ == '__main__': - main(sys.argv[1:]) + main() diff --git a/python/nav/bin/radiusparser.py b/python/nav/bin/radiusparser.py index cd9873df6c..2915f151df 100755 --- a/python/nav/bin/radiusparser.py +++ b/python/nav/bin/radiusparser.py @@ -52,7 +52,15 @@ my_logfile = "./radiusparser.log" # Location of this program's debug log file -def main(args): +def main(args=None): + # If script is already running, abort + if pid_running(): + sys.exit(1) + else: + print("Running instance of script not found. Starting...") + + if args == None: + args = sys.argv[1:] try: db_params = (dbhost, dbport, dbname, dbuser, dbpasswd) @@ -460,10 +468,4 @@ def iter_lines(file): ########### if __name__ == '__main__': - # If script is already running, abort - if pid_running(): - sys.exit(1) - else: - print("Running instance of script not found. Starting...") - - main(sys.argv[1:]) + main() diff --git a/python/nav/bin/servicemon.py b/python/nav/bin/servicemon.py index 169c095d41..08f828c0bd 100755 --- a/python/nav/bin/servicemon.py +++ b/python/nav/bin/servicemon.py @@ -139,8 +139,12 @@ def signalhandler(self, signum, _): _logger.info("Caught %s. Resuming operation.", signum) -def main(foreground): +def main(args=None): """Daemon main entry point""" + os.umask(0o0002) + if args == None: + args = parse_args() + foreground = args.foreground conf = config.serviceconf() pidfilename = conf.get("pidfile", "servicemon.pid") @@ -174,6 +178,4 @@ def parse_args(): if __name__ == '__main__': - os.umask(0o0002) - args = parse_args() - main(args.foreground) + main() diff --git a/python/nav/bin/sortedstats_cacher.py b/python/nav/bin/sortedstats_cacher.py index bce6709985..fd90607b19 100755 --- a/python/nav/bin/sortedstats_cacher.py +++ b/python/nav/bin/sortedstats_cacher.py @@ -18,7 +18,7 @@ _logger = logging.getLogger('nav.sortedstats_cacher') -def main(timeframe, config): +def run(timeframe, config): _logger.info("Running for timeframe %s", timeframe) reports = config.get_reports(timeframe) for report_name, report in reports.items(): @@ -48,11 +48,15 @@ def exit_if_running(pidfile): exit(1) -if __name__ == '__main__': +def main(): init_generic_logging(logfile=LOGFILE, stderr=False, read_config=True) timeframe = get_parser().parse_args().timeframe pidfile = f"sortedstats_cacher_{timeframe}.pid" exit_if_running(pidfile) writepidfile(pidfile) config = SortedStatsConfig() - main(timeframe, config) + run(timeframe, config) + + +if __name__ == '__main__': + main() diff --git a/python/nav/bin/start_arnold.py b/python/nav/bin/start_arnold.py index a834933a29..4cc7968d08 100755 --- a/python/nav/bin/start_arnold.py +++ b/python/nav/bin/start_arnold.py @@ -63,9 +63,12 @@ _logger = logging.getLogger('nav.start_arnold') -def main(args): +def main(args=None): """Main controller""" + if args == None: + args = parse_command_options() + init_generic_logging(logfile=LOG_FILE, stderr=False, read_config=True) if args.listblocktypes: @@ -316,5 +319,4 @@ def valid_profile(detention_profile_id): if __name__ == '__main__': - _args = parse_command_options() - main(_args) + main() diff --git a/setup.py b/setup.py index d3f92f131f..c6e74831b0 100644 --- a/setup.py +++ b/setup.py @@ -3,20 +3,12 @@ from setuptools.command.build import build -def find_scripts(): - for candidate in glob('bin/*'): - with open(candidate) as handle: - if handle.readline().startswith("#!"): - yield candidate - - # Ensure CSS files are built every time build is invoked build.sub_commands = [('build_sass', None)] + build.sub_commands setup( setup_requires=['libsass', 'setuptools_scm'], - scripts=list(find_scripts()), sass_manifests={ 'nav.web': { 'sass_path': 'sass', diff --git a/tests/integration/bin_test.py b/tests/integration/bin_test.py index 72a7fb3920..1c07a8908e 100644 --- a/tests/integration/bin_test.py +++ b/tests/integration/bin_test.py @@ -3,6 +3,9 @@ import subprocess +BINDIR = './python/nav/bin' + + def test_binary_runs(binary): """Verifies that a command runs with a zero exit code""" proc = subprocess.Popen(binary, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) @@ -23,7 +26,7 @@ def test_naventity_runs_without_error_with_arguments(localhost, snmpsim): Added in regards to: https://github.com/Uninett/nav/issues/2433 """ proc = subprocess.Popen( - ["./bin/naventity", localhost.ip, "-p", "1024"], + [BINDIR + "/naventity.py", localhost.ip, "-p", "1024"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, ) @@ -45,7 +48,7 @@ def test_nav_runs_without_error_without_arguments(): Added in regards to: https://github.com/Uninett/nav/issues/2601 """ proc = subprocess.Popen( - ["./bin/nav"], + [BINDIR + "/nav.py"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 03f73a9e80..65c7a13c62 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -86,7 +86,7 @@ def stop_gunicorn(): r'^# +-\*-\s*testargs:\s*(?P.*?)\s*(-\*-)?\s*$', re.MULTILINE ) NOTEST_PATTERN = re.compile(r'^# +-\*-\s*notest\s*(-\*-)?\s*$', re.MULTILINE) -BINDIR = './bin' +BINDIR = './python/nav/bin' def pytest_generate_tests(metafunc): @@ -119,6 +119,7 @@ def _is_excluded(filename): return ( filename.endswith('~') or filename.startswith('.') + or filename.startswith('__') or filename.startswith('Makefile') ) From f9c6a5ddff0acae95d84e52c07256e26989e7b51 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Fri, 17 Nov 2023 13:21:23 +0100 Subject: [PATCH 03/15] Rename nav.py script-file to navmain.py This makes it possible for all the other scripts to find nav itself. It will still be installed in a bin-dir as "nav". --- pyproject.toml | 2 +- python/nav/bin/{nav.py => navmain.py} | 5 +++-- tests/integration/bin_test.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) rename python/nav/bin/{nav.py => navmain.py} (98%) diff --git a/pyproject.toml b/pyproject.toml index fdd4c7ea39..d93bda62c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ logengine = "nav.bin.logengine:main" macwatch = "nav.bin.macwatch:main" mailin = "nav.bin.mailin:main" maintengine = "nav.bin.maintengine:main" -nav = "nav.bin.nav:main" +nav = "nav.bin.navmain:main" navcheckservice = "nav.bin.navcheckservice:main" navclean = "nav.bin.navclean:main" navdf = "nav.bin.navdf:main" diff --git a/python/nav/bin/nav.py b/python/nav/bin/navmain.py similarity index 98% rename from python/nav/bin/nav.py rename to python/nav/bin/navmain.py index e032ba293b..b69c5bce93 100755 --- a/python/nav/bin/nav.py +++ b/python/nav/bin/navmain.py @@ -23,9 +23,10 @@ import argparse import textwrap -from .. import colors +from nav import colors + try: - from ..startstop import ServiceRegistry, CommandFailedError, CrontabError + from nav.startstop import ServiceRegistry, CommandFailedError, CrontabError except ImportError: print( "Fatal error: Could not find the nav.startstop module.\nIs your " diff --git a/tests/integration/bin_test.py b/tests/integration/bin_test.py index 1c07a8908e..5e8947fb2f 100644 --- a/tests/integration/bin_test.py +++ b/tests/integration/bin_test.py @@ -48,7 +48,7 @@ def test_nav_runs_without_error_without_arguments(): Added in regards to: https://github.com/Uninett/nav/issues/2601 """ proc = subprocess.Popen( - [BINDIR + "/nav.py"], + [BINDIR + "/navmain.py"], stderr=subprocess.STDOUT, stdout=subprocess.PIPE, ) From 303d14975ec61d59742861c51ac0526ca843f1a9 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 20 Nov 2023 11:00:09 +0100 Subject: [PATCH 04/15] Update hardcoded paths in tests --- tests/integration/pping_test.py | 9 +++++++-- tests/integration/smsd_test.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/integration/pping_test.py b/tests/integration/pping_test.py index 61b6945da5..6152e34cfa 100644 --- a/tests/integration/pping_test.py +++ b/tests/integration/pping_test.py @@ -2,6 +2,7 @@ various pping integration tests """ import os +from pathlib import Path import getpass from shutil import which @@ -17,6 +18,9 @@ from nav.config import find_config_file +BINDIR = './python/nav/bin' + + def can_be_root(): try: get_root_method() @@ -86,8 +90,9 @@ def get_pping_output(timeout=5): Also asserts that pping shouldn't unexpectedly exit with a zero exitcode. """ - pping = which('pping.py') - assert pping, "Cannot find pping.py on path" + pping = Path(BINDIR).absolute() / 'pping.py' + assert pping.exists(), "Cannot find pping.py on path" + pping = str(pping) cmd = get_root_method() + ["/usr/bin/timeout", str(timeout), pping, "-f"] try: output = check_output(cmd, stderr=STDOUT) diff --git a/tests/integration/smsd_test.py b/tests/integration/smsd_test.py index 93bd6cf9ad..cc09a9f8fb 100644 --- a/tests/integration/smsd_test.py +++ b/tests/integration/smsd_test.py @@ -2,6 +2,7 @@ smsd integration tests """ import os +import os.path try: from subprocess32 import STDOUT, check_output, TimeoutExpired, CalledProcessError @@ -13,6 +14,8 @@ from nav.config import find_config_file, find_config_dir +BINDIR = "./python/nav/bin" + def test_smsd_test_message_with_uninettmaildispatcher_should_work( smsd_test_config, @@ -89,7 +92,7 @@ def get_smsd_test_output(phone_no, timeout=5): returns the combined stdout+stderr output from the process. """ - cmd = ['smsd.py', '-t', phone_no] + cmd = [os.path.join(BINDIR, 'smsd.py'), '-t', phone_no] try: # remove environment var that may interfere with our test - this mimicks # a production environment From 0fea24e385376c1de660118e2328ebddcd6f130b Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 20 Nov 2023 11:44:55 +0100 Subject: [PATCH 05/15] Fix ipdevpolld-script All scripts installed via entrypoints needs to have an importable callable. This script lacked that. --- python/nav/bin/ipdevpolld.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/nav/bin/ipdevpolld.py b/python/nav/bin/ipdevpolld.py index 50f13b9508..cd61f198e1 100755 --- a/python/nav/bin/ipdevpolld.py +++ b/python/nav/bin/ipdevpolld.py @@ -16,7 +16,8 @@ bootstrap_django(__file__) -if __name__ == '__main__': + +def main(): if platform.system() == "Linux": from nav.ipdevpoll.epollreactor2 import install @@ -25,3 +26,7 @@ from nav.ipdevpoll.daemon import main main() + + +if __name__ == '__main__': + main() From a248b63828b4a9c026926eebb0bf810145f78e47 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 20 Nov 2023 12:16:02 +0100 Subject: [PATCH 06/15] Move main function as per nitpick --- python/nav/bin/sortedstats_cacher.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python/nav/bin/sortedstats_cacher.py b/python/nav/bin/sortedstats_cacher.py index fd90607b19..2e835a9fc5 100755 --- a/python/nav/bin/sortedstats_cacher.py +++ b/python/nav/bin/sortedstats_cacher.py @@ -18,6 +18,16 @@ _logger = logging.getLogger('nav.sortedstats_cacher') +def main(): + init_generic_logging(logfile=LOGFILE, stderr=False, read_config=True) + timeframe = get_parser().parse_args().timeframe + pidfile = f"sortedstats_cacher_{timeframe}.pid" + exit_if_running(pidfile) + writepidfile(pidfile) + config = SortedStatsConfig() + run(timeframe, config) + + def run(timeframe, config): _logger.info("Running for timeframe %s", timeframe) reports = config.get_reports(timeframe) @@ -48,15 +58,5 @@ def exit_if_running(pidfile): exit(1) -def main(): - init_generic_logging(logfile=LOGFILE, stderr=False, read_config=True) - timeframe = get_parser().parse_args().timeframe - pidfile = f"sortedstats_cacher_{timeframe}.pid" - exit_if_running(pidfile) - writepidfile(pidfile) - config = SortedStatsConfig() - run(timeframe, config) - - if __name__ == '__main__': main() From 518be2d687c1b9d1d4e2bad63c9bac61204889a1 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 20 Nov 2023 12:23:22 +0100 Subject: [PATCH 07/15] Update script-names in cron-files --- python/nav/etc/cron.d/activeip | 2 +- python/nav/etc/cron.d/emailreports | 6 +++--- python/nav/etc/cron.d/logengine | 4 ++-- python/nav/etc/cron.d/mactrace | 2 +- python/nav/etc/cron.d/maintengine | 2 +- python/nav/etc/cron.d/netbiostracker | 2 +- python/nav/etc/cron.d/sortedstats_cacher | 8 ++++---- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/python/nav/etc/cron.d/activeip b/python/nav/etc/cron.d/activeip index 1f96c0d9c0..55562167cd 100644 --- a/python/nav/etc/cron.d/activeip +++ b/python/nav/etc/cron.d/activeip @@ -1,2 +1,2 @@ ## info: Collect active ip-adresses for all prefixes -*/30 * * * * collect_active_ip.py +*/30 * * * * collect_active_ip diff --git a/python/nav/etc/cron.d/emailreports b/python/nav/etc/cron.d/emailreports index f820e83a34..b9b3833ad7 100644 --- a/python/nav/etc/cron.d/emailreports +++ b/python/nav/etc/cron.d/emailreports @@ -1,4 +1,4 @@ ## info: Send email reports for device and link availability -0 5 1 * * emailreports.py month -0 5 * * mon emailreports.py week -0 5 * * * emailreports.py day +0 5 1 * * emailreports month +0 5 * * mon emailreports week +0 5 * * * emailreports day diff --git a/python/nav/etc/cron.d/logengine b/python/nav/etc/cron.d/logengine index c7e6cf215a..5dc7e8b50f 100644 --- a/python/nav/etc/cron.d/logengine +++ b/python/nav/etc/cron.d/logengine @@ -1,10 +1,10 @@ ## info: Regularly check the syslog for network messages and update the logger database # Regular run -* * * * * logengine.py -q +* * * * * logengine -q # Delete old messages once a day -3 3 * * * logengine.py -d +3 3 * * * logengine -d # Delete old ipdevpoll job log entries once every hour 3 * * * * ipdevpolld --clean diff --git a/python/nav/etc/cron.d/mactrace b/python/nav/etc/cron.d/mactrace index ff5b346c61..339f4fa012 100644 --- a/python/nav/etc/cron.d/mactrace +++ b/python/nav/etc/cron.d/mactrace @@ -1,2 +1,2 @@ ## info: Checks NAV's cam log for watched MAC addresses -11,26,41,56 * * * * macwatch.py +11,26,41,56 * * * * macwatch diff --git a/python/nav/etc/cron.d/maintengine b/python/nav/etc/cron.d/maintengine index 63a36472f0..d3c1dc4cbf 100644 --- a/python/nav/etc/cron.d/maintengine +++ b/python/nav/etc/cron.d/maintengine @@ -1,2 +1,2 @@ ## info: Regularly check the maintenance-queue and send events to eventq -*/5 * * * * maintengine.py +*/5 * * * * maintengine diff --git a/python/nav/etc/cron.d/netbiostracker b/python/nav/etc/cron.d/netbiostracker index 14969c036c..b4c42adedf 100644 --- a/python/nav/etc/cron.d/netbiostracker +++ b/python/nav/etc/cron.d/netbiostracker @@ -1,2 +1,2 @@ ## info: Regularly fetch netbiosnames from active computers -*/15 * * * * netbiostracker.py +*/15 * * * * netbiostracker diff --git a/python/nav/etc/cron.d/sortedstats_cacher b/python/nav/etc/cron.d/sortedstats_cacher index 993826e07f..a1bdca126a 100644 --- a/python/nav/etc/cron.d/sortedstats_cacher +++ b/python/nav/etc/cron.d/sortedstats_cacher @@ -1,5 +1,5 @@ ## info: Regularly update cache for sortedstats -*/15 * * * * sortedstats_cacher.py hour -0 * * * * sortedstats_cacher.py day -0 */6 * * * sortedstats_cacher.py week -0 0 * * * sortedstats_cacher.py month +*/15 * * * * sortedstats_cacher hour +0 * * * * sortedstats_cacher day +0 */6 * * * sortedstats_cacher week +0 0 * * * sortedstats_cacher month From afc2d44786659f28d38b8a7578f9edc104a1c819 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 20 Nov 2023 12:25:49 +0100 Subject: [PATCH 08/15] Update script-names in config-files --- python/nav/etc/daemons.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/nav/etc/daemons.yml b/python/nav/etc/daemons.yml index 1424a04fd6..59188497ae 100644 --- a/python/nav/etc/daemons.yml +++ b/python/nav/etc/daemons.yml @@ -2,7 +2,7 @@ daemons: alertengine: description: Dispatches alerts according to individual alert profiles. - command: alertengine.py + command: alertengine eventengine: command: eventengine @@ -15,18 +15,18 @@ daemons: pping: description: Pings all IP devices for status monitoring. - command: pping.py + command: pping privileged: true servicemon: description: Monitors configured services. - command: servicemon.py + command: servicemon smsd: description: Dispatches queued SMS alerts. - command: smsd.py + command: smsd snmptrapd: description: Receives and processes SNMP traps and notifications. - command: snmptrapd.py + command: snmptrapd privileged: true From 5c10fa3f0b308a41a3725978801b8a769a7e6305 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 20 Nov 2023 12:33:25 +0100 Subject: [PATCH 09/15] Update text references to scripts --- python/nav/bin/autoenable.py | 2 +- python/nav/bin/start_arnold.py | 2 +- python/nav/bin/t1000.py | 2 +- python/nav/etc/snmptrapd.conf | 2 +- python/nav/web/macwatch/models.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/nav/bin/autoenable.py b/python/nav/bin/autoenable.py index 5fba3920e7..dcccb4c31c 100755 --- a/python/nav/bin/autoenable.py +++ b/python/nav/bin/autoenable.py @@ -19,7 +19,7 @@ Autoenable removes detention for computers that are done with detention. Usage: -sudo -u $NAV_USER ./autoenable.py +sudo -u $NAV_USER autoenable autoenable is meant to be run as a cronjob. It checks the configured arnold-database for any detained ports and opens them if they have a diff --git a/python/nav/bin/start_arnold.py b/python/nav/bin/start_arnold.py index 4cc7968d08..4538bdda4c 100755 --- a/python/nav/bin/start_arnold.py +++ b/python/nav/bin/start_arnold.py @@ -19,7 +19,7 @@ """ Use this to run automatic detentions based on detention profiles -Usage: start_arnold.py [options] id +Usage: start_arnold [options] id Pipe in ip-addresses to block or use the -f option to specify file Options: diff --git a/python/nav/bin/t1000.py b/python/nav/bin/t1000.py index 3dc300e5ee..46275ed922 100755 --- a/python/nav/bin/t1000.py +++ b/python/nav/bin/t1000.py @@ -19,7 +19,7 @@ t1000 finds blocked computers that have moved and blocks them again. Usage: -sudo -u $NAV_USER ./t1000.py +sudo -u $NAV_USER t1000 t1000 is meant to be run as a cronjob. It checks the database for any detained ports. If it finds any, it checks if the mac-address is active on diff --git a/python/nav/etc/snmptrapd.conf b/python/nav/etc/snmptrapd.conf index 800a53dd67..9320eccad9 100644 --- a/python/nav/etc/snmptrapd.conf +++ b/python/nav/etc/snmptrapd.conf @@ -1,4 +1,4 @@ -# Configfile for snmptrapd.py +# Configfile for snmptrapd [snmptrapd] # Each of these Python modules will be offered the chance to handle an incoming diff --git a/python/nav/web/macwatch/models.py b/python/nav/web/macwatch/models.py index b14803d43f..02a4c94141 100644 --- a/python/nav/web/macwatch/models.py +++ b/python/nav/web/macwatch/models.py @@ -26,7 +26,7 @@ class MacWatch(models.Model): """Data-model for mac-address that should get watched - by bin/macwatch.py""" + by bin/macwatch""" MAC_ADDR_DELIM_CHAR = ':' From cf4cf835ea5af6f0809934c3ddf0eb61e3e37267 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Mon, 20 Nov 2023 12:58:15 +0100 Subject: [PATCH 10/15] Update script-names in the documentation --- doc/reference/arnold.rst | 46 ++++++++++++++++---------------- doc/reference/mailin.rst | 4 +-- doc/reference/radius-install.rst | 8 +++--- doc/reference/radius.rst | 28 +++++++++---------- doc/reference/smsd.rst | 2 +- doc/reference/snmptrapd.rst | 4 +-- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/doc/reference/arnold.rst b/doc/reference/arnold.rst index 5d1caad7f1..f99d53ce67 100644 --- a/doc/reference/arnold.rst +++ b/doc/reference/arnold.rst @@ -54,15 +54,15 @@ several views available. They are as follows: In addition you have two actions you can use - *Search* and *Manual detention*. - **Search** lets you search for detentions given some search parameters. -- **Manual detention** lets you manually detain a port given an IP- or +- **Manual detention** lets you manually detain a port given an IP- or MAC-address. Predefined detentions --------------------- -The only way to use a predefined detention is by using the ``start_arnold.py`` +The only way to use a predefined detention is by using the ``start_arnold`` shell script. After creating a predefined detention you usually want to create a cron-job for running the script with some input parameters. See section about -`start_arnold.py`_. +`start_arnold`_. Using the scripts @@ -71,48 +71,48 @@ Using the scripts Arnold consists of three scripts, which all are located in the ``nav/bin`` directory. -- **autoenable.py** enables ports based on the autoenable variable available for +- **autoenable** enables ports based on the autoenable variable available for both manual and predefined detentions. -- **start_arnold.py** is used in combination with predefined detentions to +- **start_arnold** is used in combination with predefined detentions to invoke a series of detentions. -- **t1000.py** verifies that the MAC-addresses that should be offline are not +- **t1000** verifies that the MAC-addresses that should be offline are not active on other ports. If a detained MAC-address is online on another port, it will try to detain it there aswell. More details about the different scripts can be seen below. -autoenable.py -------------- +autoenable +---------- -*autoenable.py* fetches all detained ports with an autoenable-value and enables +*autoenable* fetches all detained ports with an autoenable-value and enables each of those detentions if the time is due. It can be run manually or as a periodic cron job. The simplest way of running automatic enabling periodically is to create a file -containing cron configuration that calls the *autoenable.py* program as often as +containing cron configuration that calls the *autoenable* program as often as you would like:: - 0 * * * * some_prefix/nav/bin/autoenable.py # Run every hour on the hour + 0 * * * * some_prefix/nav/bin/autoenable # Run every hour on the hour Save this snippet in a file called ``autoenable`` in NAV's ``etc/cron.d/`` directory. That way, you can add it to the navcron user's crontab by calling ``nav start autoenable``. -start_arnold.py ---------------- +start_arnold +------------ -When a predefined detention is created you can use *start_arnold.py* to invoke a +When a predefined detention is created you can use *start_arnold* to invoke a series of detentions based on the input to the script. If the file or list of addresses exist locally then you can pipe it in using for instance ``cat``:: - - # cat scanresult.txt | nav/bin/start_arnold_py -i + + # cat scanresult.txt | nav/bin/start_arnold -i or you can do it from a remote server using ssh commands:: - # cat scanresult.txt | ssh scanner@navinstall.network.com:nav/bin/start_arnold_py -i - + # cat scanresult.txt | ssh scanner@navinstall.network.com:nav/bin/start_arnold -i + To avoid having to type passwords you want to create public keys, like described for instance `here `_. @@ -123,19 +123,19 @@ Each line in this file is assumed to consist of an IP- or MAC-address and optionally a comment (separated by a space). For each valid address a detention will be made. Lines starting with *#* will be skipped. -t1000.py --------- +t1000 +----- -This script needs to be set up to run in the same way as `autoenable.py`_. +This script needs to be set up to run in the same way as `autoenable`_. -*t1000.py* fetches all detained ports and checks if the MAC-address which was +*t1000* fetches all detained ports and checks if the MAC-address which was behind the detained port is active on another port. If it is, it enforces the detention on that port aswell. Depending on options given at detention-time it will either remove the detention on the old port or just leave it. .. warning:: This does not detain the new port immediately after a detained computer has moved to it, because it takes some time before NAV discovers the - new location of the MAC-address. This combined with the interval t1000.py + new location of the MAC-address. This combined with the interval ``t1000`` runs in could give the user quite some time with access before being detained again. This on-and-off behavior of internet access has been known to cause confusion and annoyance among the users - use this script knowing that. diff --git a/doc/reference/mailin.rst b/doc/reference/mailin.rst index 498e7acd89..693d5c3a4a 100644 --- a/doc/reference/mailin.rst +++ b/doc/reference/mailin.rst @@ -24,11 +24,11 @@ connections from outside the server, or no messages will come through. Pick an e-mail address on your NAV server to send 3rd party alerts to, for example `mailin@nav.example.org`. Mail received at this address should be -piped through the ``mailin.py`` program. This can usually be accomplished by +piped through the ``mailin`` program. This can usually be accomplished by adding an e-mail alias to :file:`/etc/aliases`, like this:: cat >> /etc/aliases < Date: Thu, 29 Feb 2024 13:02:13 +0100 Subject: [PATCH 11/15] Explicitly use python3 --- .github/workflows/build-and-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 4560c11d73..0155b47feb 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -49,7 +49,7 @@ jobs: - name: "Install test runner dependencies" run: | set -xe - python -m pip install --upgrade 'pip==23.1.0' setuptools wheel 'tox<4' tox-gh-actions coverage virtualenv snmpsim 'pyasn1<0.5.0' + python3 -m pip install --upgrade 'pip==23.1.0' setuptools wheel 'tox<4' tox-gh-actions coverage virtualenv snmpsim 'pyasn1<0.5.0' sudo apt-get install -y nbtscan # virtualenv seems to currently be embedding a broken version of @@ -80,7 +80,7 @@ jobs: export DISPLAY=:99 ulimit -c unlimited sudo sysctl -w kernel.core_pattern=/tmp/core-%e.%p.%h.%t - python -m tox + python3 -m tox - name: Upload Selenium driver logs (${{ matrix.python-version }}) if: always() From 04c1fa59e022ef5ee56c112afc016bf2a85ba050 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 29 Feb 2024 17:01:32 +0000 Subject: [PATCH 12/15] Add sudo to tests/docker This makes sudo available in the Docker-based test environment, in order to make the environment slightly more similar to GitHub Actions (which doesn't feature gosu). With sudo it is easier to locally reproduce sudo-related problems observed on GitHub actions. --- tests/docker/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 650f5aa1ed..6add9e6052 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -10,9 +10,12 @@ ENV DEBIAN_FRONTEND noninteractive ### Installing packages RUN sed -ie 's/^# *deb-src/deb-src/' /etc/apt/sources.list # Enable source repos RUN apt-get update && \ - apt-get install -y software-properties-common gosu && \ + apt-get install -y software-properties-common gosu sudo && \ chmod u+s /usr/sbin/gosu +RUN echo "build ALL =(ALL: ALL) NOPASSWD: ALL" > /etc/sudoers.d/navbuild + + RUN add-apt-repository ppa:deadsnakes/ppa && \ apt-get update && \ apt-get -y install --no-install-recommends \ From 7dbda65d0768b5064f3ed77b8f34f2dff103fdae Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 29 Feb 2024 17:16:11 +0000 Subject: [PATCH 13/15] Find installed pping executable under its new name --- tests/integration/pping_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/pping_test.py b/tests/integration/pping_test.py index 6152e34cfa..8068b6c008 100644 --- a/tests/integration/pping_test.py +++ b/tests/integration/pping_test.py @@ -90,9 +90,8 @@ def get_pping_output(timeout=5): Also asserts that pping shouldn't unexpectedly exit with a zero exitcode. """ - pping = Path(BINDIR).absolute() / 'pping.py' - assert pping.exists(), "Cannot find pping.py on path" - pping = str(pping) + pping = which("pping") + assert pping, "Cannot find pping in PATH" cmd = get_root_method() + ["/usr/bin/timeout", str(timeout), pping, "-f"] try: output = check_output(cmd, stderr=STDOUT) From ee67ac3caaee6dbe32d24f6e61d3bf376b779162 Mon Sep 17 00:00:00 2001 From: Hanne Moa Date: Thu, 7 Mar 2024 10:23:02 +0100 Subject: [PATCH 14/15] Add release note about changed command names Co-authored-by: Morten Brekkevold --- NOTES.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/NOTES.rst b/NOTES.rst index c356b9d6cd..ba8b2fbe33 100644 --- a/NOTES.rst +++ b/NOTES.rst @@ -11,6 +11,38 @@ please go to https://github.com/uninett/nav/milestones . NAV 5.9 ======= +Changed names of NAV command line programs +------------------------------------------ +NAV has switched to a more canonical way of installing Python command line +scripts, or "binaries". This means that all NAV command line programs that +previously ended with a ``.py`` extension now have been stripped of that +extension. Any custom cron jobs or scripts you have that may reference these +NAV commands must be updated in order to continue working. + +It also means that you need to make sure your :file:`daemons.yml` configuration +file is up-to-date after an upgrade, as well as the NAV cronjob snippets in the +:file:`cron.d/` configuration directory. + +These commands are affected and no longer have a ``.py`` extension: + +* ``alertengine`` +* ``autoenable`` +* ``collect_active_ip`` +* ``emailreports`` +* ``logengine`` +* ``macwatch`` +* ``mailin`` +* ``maintengine`` +* ``netbiostracker`` +* ``pping`` +* ``radiusparser`` +* ``servicemon`` +* ``smsd`` +* ``snmptrapd`` +* ``sortedstats_cacher`` +* ``start_arnold`` +* ``t1000`` + Web security ------------ From 300d8917a1b46db8f6f611d280bd0dad74c184c0 Mon Sep 17 00:00:00 2001 From: Morten Brekkevold Date: Thu, 7 Mar 2024 10:01:17 +0100 Subject: [PATCH 15/15] Remove `bin/` from src code dir layout docs --- doc/hacking/hacking.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/hacking/hacking.rst b/doc/hacking/hacking.rst index cf2c9d3818..e24b9387a5 100644 --- a/doc/hacking/hacking.rst +++ b/doc/hacking/hacking.rst @@ -113,7 +113,6 @@ A rough guide to the source tree: ================================= ================================================================= Directory Description ================================= ================================================================= -:file:`bin/` NAV 'binaries'; executable scripts and programs. :file:`contrib/` User contributed NAV tools. NAV doesn't depend on these, and any maintenance of them is left up to the original developers. We do not offer support for these tools.