Move to Python 3 #1316

garberg · 2022-11-10T22:31:21Z

It's way overdue, but a move to Python 3. Based on the work in #1253.

nipap-cli/nipap_cli/__init__.py

nipap-www/nipapwww/__init__.py

nipap/nipap/__init__.py

nipap/nipap/authlib.py

nipap/nipap/daemon.py

nipap/nipap/nipapconfig.py

nipap/nipap/smart_parsing.py

houndci-bot · 2022-11-10T23:17:40Z

nipap/nipap/smart_parsing.py

-        comp_ipv4_address = Combine(ipv4_oct + ('.' + ipv4_oct*3))
-        ipv4_address = Combine(ipv4_oct + ('.' + ipv4_oct*3)).setResultsName('ipv4_address')
+        comp_ipv4_address = Combine(ipv4_oct + ('.' + ipv4_oct * 3))
+        ipv4_address = Combine(ipv4_oct + ('.' + ipv4_oct * 3)).setResultsName('ipv4_address')


line too long (94 > 79 characters)
local variable 'ipv4_address' is assigned to but never used

houndci-bot · 2022-11-10T23:17:40Z

nipap/nipap/smart_parsing.py


-from errors import *
+from .errors import *


'.errors.*' imported but unused
'from .errors import *' used; unable to detect undefined names

houndci-bot · 2022-11-10T23:17:40Z

nipap/nipap/smart_parsing.py

 import logging
 import re

 import IPy

-from pyparsing import Combine, Forward, Group, Literal, nestedExpr, OneOrMore, ParseResults, quotedString, Regex, QuotedString, Word, ZeroOrMore, alphanums, nums, oneOf
+from pyparsing import Combine, Forward, Group, Literal, nestedExpr, OneOrMore, ParseResults, quotedString, Regex, \


'pyparsing.OneOrMore' imported but unused
'pyparsing.alphanums' imported but unused
Black would make changes.
line too long (115 > 79 characters)

nipap/nipap/xmlrpc.py

tests/nipaptest.py

houndci-bot · 2022-11-11T08:13:35Z

tests/test_cli.py


 server_url = "http://unittest:[email protected]:1337/XMLRPC"
-s = xmlrpclib.Server(server_url, allow_none=1);
+s = xmlrpc.client.Server(server_url, allow_none=1);


statement ends with a semicolon

houndci-bot · 2022-11-11T08:13:35Z

tests/test_cli.py

@@ -17,10 +17,10 @@
 logger.setLevel(logging.DEBUG)
 log_format = "%(levelname)-8s %(message)s"

-import xmlrpclib
+import xmlrpc.client


module level import not at top of file

nipap/setup.py

nipap/xmlbench.py

houndci-bot · 2022-11-11T08:16:02Z

nipap/xmlbench.py


    def setupReqs(self):
-        for i in range(0,self.concurrent_reqs): # make the initial pool of requests
+        for i in range(0, self.concurrent_reqs):  # make the initial pool of requests


Loop control variable 'i' not used within the loop body. If this is intended, start the name with an underscore.
line too long (85 > 79 characters)

nipap/nipap/xmlrpc.py

houndci-bot · 2022-11-11T08:16:03Z

nipap/nipap/xmlrpc.py

-            raise Fault(1000, ("Function argument must be XML-RPC struct/Python dict (Python %s given)." %
-                type(nipap_args).__name__ ))
+            raise Fault(1000, ("Function argument must be XML-RPC struct/Python dict (Python {} given).".format(
+                type(nipap_args).__name__ )))


whitespace before ')'

nipap/nipap/xmlrpc.py

nipap/nipap/smart_parsing.py

nipap/nipap/xmlrpc.py

houndci-bot · 2022-11-11T14:52:15Z

nipap/nipap/xmlrpc.py


-from nipapconfig import NipapConfig
-from backend import Nipap, NipapError
+from .nipapconfig import NipapConfig


'.nipapconfig.NipapConfig' imported but unused

houndci-bot · 2022-11-11T14:52:15Z

nipap/nipap/xmlrpc.py

 from flask import request, Response
-from flaskext.xmlrpc import XMLRPCHandler, Fault
+from flask_xmlrpcre.xmlrpcre import XMLRPCHandler, Fault
+from flask_compress import Compress


'flask_compress.Compress' imported but unused

houndci-bot · 2022-11-11T14:52:15Z

nipap/nipap/xmlrpc.py

 from functools import wraps
-from flask import Flask
+from flask import Flask, current_app


'flask.Flask' imported but unused

nipap/xml-test.py

houndci-bot · 2022-11-11T22:58:23Z

tests/test_xmlrpc.py

@@ -1402,7 +1402,7 @@ def test_edit_pool(self):
        expected['tags'] = []
        expected['avps'] = {}

-        self.assertEquals(self._mangle_pool_result(s.list_pool({ 'auth': ad,
+        self.assertEqual(self._mangle_pool_result(s.list_pool({ 'auth': ad,


whitespace after '{'

tests/test_xmlrpc.py

garberg · 2022-12-08T08:30:19Z

As the authentication library is shared between nipapd and nipap-www, Python 3 support needs to be fixed for nipap-www as well before this can be merged. Unfortunately Pylons, the web framework used, lacks Python 3 support and is abandoned. To prepare for this, a migration from Pylons to Flask has been initiated.

nipap/nipap/xmlrpc.py

houndci-bot · 2023-07-05T12:59:48Z

whoisd/nipap_whoisd.py

@@ -182,4 +182,4 @@ def drop_privileges(uid_name='nobody', gid_name='nogroup'):
    os.setuid(uid)

    # Ensure a very conservative umask
-    old_umask = os.umask(077)
+    old_umask = os.umask(0o77)


local variable 'old_umask' is assigned to but never used

houndci-bot · 2023-07-05T12:59:49Z

whoisd/nipap_whoisd.py

-      except OSError, e:
-         raise Exception, "%s [%d]" % (e.strerror, e.errno)
+      except OSError as e:
+         raise Exception("%s [%d]" % (e.strerror, e.errno))


indentation is not a multiple of four

houndci-bot · 2023-07-05T12:59:49Z

whoisd/nipap_whoisd.py

@@ -85,8 +85,8 @@ def createDaemon():
         # longer a session leader, preventing the daemon from ever acquiring
         # a controlling terminal.
         pid = os.fork()    # Fork a second child.
-      except OSError, e:
-         raise Exception, "%s [%d]" % (e.strerror, e.errno)
+      except OSError as e:


indentation is not a multiple of four

houndci-bot · 2023-07-05T12:59:49Z

whoisd/nipap_whoisd.py

-   except OSError, e:
-      raise Exception, "%s [%d]" % (e.strerror, e.errno)
+   except OSError as e:
+      raise Exception("%s [%d]" % (e.strerror, e.errno))


indentation is not a multiple of four

tests/test_xmlrpc.py

houndci-bot · 2023-07-05T12:59:51Z

tests/test_xmlrpc.py

@@ -993,7 +993,7 @@ def test_prefix_from_pool(self):
        child = s.add_prefix({ 'auth': ad, 'attr': prefix_attr, 'args': args })
        #expected['id'] = child['id']
        #p = s.list_prefix({ 'auth': ad, 'attr': { 'id': child['id'] } })[1]
-        #self.assertEquals(p, expected)
+        #self.assertEqual(p, expected)


block comment should start with '# '

tests/test_xmlrpc.py

houndci-bot · 2023-07-05T13:11:12Z

tests/test_rest.py

@@ -192,17 +192,17 @@ def test_prefix_add(self):

        request = requests.post(self.server_url, headers=self.headers, json = attr)
        text = request.text
-        self.assertRegexpMatches(text,"'attr' must be a dict")
+        self.assertRegex(text,"'attr' must be a dict")


missing whitespace after ','

tests/test_xmlrpc.py

houndci-bot · 2023-07-05T13:11:12Z

tests/test_xmlrpc.py


        # smart search
        res = self._mangle_vrf_result(s.smart_search_vrf({ 'auth': ad, 'query_string': 'forwarding instance' }))
-        self.assertEquals(res['result'], [ attr, ], 'Smart search result did not match')
+        self.assertEqual(res['result'], [ attr, ], 'Smart search result did not match')


line too long (87 > 79 characters)
whitespace after '['

houndci-bot · 2023-07-05T13:11:13Z

tests/test_xmlrpc.py

@@ -461,11 +461,11 @@ def test_vrf_add_search(self):
            'val2': 'instance 65000'
        }
        res = self._mangle_vrf_result(s.search_vrf({ 'auth': ad, 'query': q }))
-        self.assertEquals(res['result'], [ attr, ], 'Search result from regex match did not match')
+        self.assertEqual(res['result'], [ attr, ], 'Search result from regex match did not match')


line too long (98 > 79 characters)
whitespace after '['

tests/test_xmlrpc.py

houndci-bot · 2023-07-05T13:11:14Z

tests/test_xmlrpc.py

@@ -354,7 +354,7 @@ def test_vrf_add_list(self):
        self.assertEqual(self._mangle_vrf_result(s.list_vrf({ 'auth': ad, 'vrf': { 'id': vrf['id'] } })), [ ref, ])

        attr['rt'] = '123:abc'
-        with self.assertRaisesRegexp(xmlrpclib.Fault, '.'): # TODO: specify exception string
+        with self.assertRaisesRegex(xmlrpc.client.Fault, '.'): # TODO: specify exception string


at least two spaces before inline comment
line too long (95 > 79 characters)

tests/test_xmlrpc.py

houndci-bot · 2023-07-05T13:11:14Z

tests/test_xmlrpc.py


 server_url = "http://unittest:[email protected]:1337/XMLRPC"
-s = xmlrpclib.Server(server_url, allow_none=1);
+s = xmlrpc.client.Server(server_url, allow_none=1);


statement ends with a semicolon

houndci-bot · 2023-07-05T13:55:20Z

tests/test_xmlrpc.py

@@ -25,10 +25,10 @@
 logger.setLevel(logging.DEBUG)
 log_format = "%(levelname)-8s %(message)s"

-import xmlrpclib
+import xmlrpc.client


module level import not at top of file

nipap-www/nipapwww/xhr.py

Move to newest or replacement libs Fix syntax to 3.6+ Overhaul strings to optimize performance and readability Update build stuff to 3.6 + move docker img to 18.04/bionic

Remove requirements which are built into Python 3.

... as it's not available in Python 3.

There was a special case in the deb package postinst action which set the psql database host to "" if it was configured as localhost. Not sure why this was needed, but "" caused psycopg2 to break in Python 3, could be due to different handling in the config parser... Trying to write nothing instead of "".

To get the tests to pass in Unbuntu 20.04 a few work arounds were needed. Over time the workarounds should be removed, but now I need progress. * Change how nipapd is started when running the apt tests. From some reason the daemon won't start with systemd, so let's just start it manually. Also, it by default tries to drop privileges and then cannot read some postgres certificate in /root. Thus, avoid dropping privileges. And yes, the daemon won't start from systemd without dropping privileges either. * Run the CLI tests over plain-text HTTP. With the newer versions of TLS libs in 20.04, a CN is not enough but a subject alternative name (SAN) is required as well. It's a bit more tricky to generate a cert with SAN, so instead I let the CLI tests run over plain-text HTTP instead. The REST tests pass over TLS (with warnings regarding missing SAN). * As the Python 2 version of NIPAP cannot be installed on Ubuntu 20.04, the upgrade tests are disabled until we can upgrade from a Python 3-based version. * Enabled an "Accident analysis"-step in the CI pipeline. If the test fails it's run and gathers some data for debugging. More debugging ci: Removed debugging ci: Cleanup for starting nipapd manually

Moved the imports in __init__.py in the nipapwww module to be able to import the module during package build without having the dependencies installed.

Fix the PID file handling. Not sure if the changes were introduced in the transition from Python 2 to 3 or later, but changes were needed as the file weren't created on start, PID wasn't visible in the error message when starting two nipapd processes and the PID file wasn't truncated before adding a new PID.

Fixed how selection of number of forks to run was handled.

Try running nipapd from apt again, after fixing handling of forks and PID file.

Updated the WSGI setup guide to instruct the user to install Python 3 version of mod_wsgi.

Surprisingly small changes to make the WWW UI Python 3 compatible.

From some reason the REST tests suddenly starts to fail over HTTPS in the CI environment. There is anoher HTTPS-related problem for XML-RPC related to SNI, but the REST tests previously passed with warnings for this. Disabling HTTPS for REST tests for now.

Revert changes in how config files were opened which was introduced in the Python 3 conversion. The change caused an uncaught exception when trying to open a non-existant config file.

The behavour of configparser seems to have changed, where the Python 2 version returned an empty string when a value in the config file was set to "", two double quotes, and the Python 3 version returns a string containing two double quotes which the PostgreSQL client library dislikes as database host. As this is the default config generated by the Debian install scripts it's probably pretty common, so a workaround was implemented where the string "" results in the previous behaviour.

garberg · 2023-10-06T07:29:57Z

LGTM, let's go! Some further things to fix before a release can be made, but we need to get this our of the door.

houndci-bot reviewed Nov 10, 2022

View reviewed changes

garberg force-pushed the py3.6-fixes branch from 269dfca to 468ce70 Compare November 10, 2022 22:38

houndci-bot reviewed Nov 10, 2022

View reviewed changes

houndci-bot reviewed Nov 11, 2022

View reviewed changes

jonkerj mentioned this pull request Dec 12, 2022

Dockerfile.nipapd Fails to build #1304

Open

garberg force-pushed the py3.6-fixes branch from b0fd2d9 to 66532c4 Compare July 5, 2023 12:59

houndci-bot reviewed Jul 5, 2023

View reviewed changes

garberg force-pushed the py3.6-fixes branch 10 times, most recently from ae9d5f6 to 5e22443 Compare October 5, 2023 12:32

houndci-bot reviewed Oct 5, 2023

View reviewed changes

nipap-www/nipapwww/xhr.py Show resolved Hide resolved

nipap-www/nipapwww/xhr.py Show resolved Hide resolved

garberg force-pushed the py3.6-fixes branch from b071654 to 2c17f60 Compare October 6, 2023 06:16

garberg changed the title ~~WIP: Move to Python 3~~ Move to Python 3 Oct 6, 2023

Port to py3.6 fixes SpriteLink#1252

4da0328

Move to newest or replacement libs Fix syntax to 3.6+ Overhaul strings to optimize performance and readability Update build stuff to 3.6 + move docker img to 18.04/bionic

garberg added 22 commits October 6, 2023 09:16

ci: Run tests on Ubuntu 20.04, install testing

a708075

ci: Run 18.04 again

fe985a4

backend: Fixed log message formatting

0493ad3

tests: Cast ID to integer before comparison

2b3d4cc

backend: Updated REST module for Py3

c6446f1

tests: Updated REST tests to Py3

8dbb85e

www: Updated imports for Python3

728ba79

nipap: Remove built-in requirements

1cea16c

Remove requirements which are built into Python 3.

nipapd: Remove unicode param to register_hstore

1f25915

... as it's not available in Python 3.

www: Updated more deb deps to python3-

8280406

test: Run tests on Ubuntu 20.04

8e79c25

www: Move imports into function

17e2bd1

Moved the imports in __init__.py in the nipapwww module to be able to import the module during package build without having the dependencies installed.

nipapd: Fix handling of number of forks

ce592f7

Fixed how selection of number of forks to run was handled.

ci: Run apt nipapd from systemctl

0bac580

Try running nipapd from apt again, after fixing handling of forks and PID file.

docs: Update WSGI setup to Python 3

9bb5a2b

Updated the WSGI setup guide to instruct the user to install Python 3 version of mod_wsgi.

www: Changes for Python 3 compatibility

501c4c9

Surprisingly small changes to make the WWW UI Python 3 compatible.

nipapd: Fix opening of missing config files

2ac35c8

Revert changes in how config files were opened which was introduced in the Python 3 conversion. The change caused an uncaught exception when trying to open a non-existant config file.

garberg force-pushed the py3.6-fixes branch from 2c17f60 to 7539743 Compare October 6, 2023 07:18

garberg merged commit 227bba1 into SpriteLink:master Oct 6, 2023
3 checks passed

This was referenced Oct 6, 2023

port nipap-www to python 3.6 #1254

Closed

port nipap-whoisd to py3.6 #1255

Closed

Run CI tests on Ubuntu 20.04 #1331

Closed

Port to py3.6 fixes #1252 #1253

Closed

garberg deleted the py3.6-fixes branch April 11, 2024 22:44

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Move to Python 3 #1316

Move to Python 3 #1316

garberg commented Nov 10, 2022

houndci-bot Nov 10, 2022

houndci-bot Nov 10, 2022

houndci-bot Nov 10, 2022

houndci-bot Nov 11, 2022

houndci-bot Nov 11, 2022

houndci-bot Nov 11, 2022

houndci-bot Nov 11, 2022

houndci-bot Nov 11, 2022

houndci-bot Nov 11, 2022

houndci-bot Nov 11, 2022

houndci-bot Nov 11, 2022

garberg commented Dec 8, 2022

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

houndci-bot Jul 5, 2023

garberg commented Oct 6, 2023

Move to Python 3 #1316

Move to Python 3 #1316

Conversation

garberg commented Nov 10, 2022

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

garberg commented Dec 8, 2022

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

garberg commented Oct 6, 2023