From a0971afa72287faab5b2ff1c7cbdf8495a754341 Mon Sep 17 00:00:00 2001 From: Martin Styk Date: Tue, 29 Oct 2024 21:06:37 +0100 Subject: [PATCH] Make integration for LC Python 2 and 3 compatible Signed-off-by: Martin Styk --- .../src/bkr/inttest/labcontroller/__init__.py | 4 +- .../labcontroller/test_distro_import.py | 16 ++-- .../inttest/labcontroller/test_provision.py | 4 +- .../bkr/inttest/labcontroller/test_proxy.py | 87 ++++++++++--------- 4 files changed, 59 insertions(+), 52 deletions(-) diff --git a/IntegrationTests/src/bkr/inttest/labcontroller/__init__.py b/IntegrationTests/src/bkr/inttest/labcontroller/__init__.py index 8b43a6d9f..5a4d930d9 100644 --- a/IntegrationTests/src/bkr/inttest/labcontroller/__init__.py +++ b/IntegrationTests/src/bkr/inttest/labcontroller/__init__.py @@ -8,13 +8,13 @@ import os import shutil import signal -from socket import gethostname -from urlparse import urlparse, urlunparse from bkr.labcontroller.config import load_conf, get_conf from turbogears.database import session from bkr.server.model import LabController, Watchdog, Recipe, RecipeSet, \ System, SystemStatus from bkr.inttest import data_setup, Process, DatabaseTestCase + + log = logging.getLogger(__name__) # XXX this should be inside setup_package, but lots of code in diff --git a/IntegrationTests/src/bkr/inttest/labcontroller/test_distro_import.py b/IntegrationTests/src/bkr/inttest/labcontroller/test_distro_import.py index fb5e99868..c392abf6d 100644 --- a/IntegrationTests/src/bkr/inttest/labcontroller/test_distro_import.py +++ b/IntegrationTests/src/bkr/inttest/labcontroller/test_distro_import.py @@ -9,13 +9,15 @@ import subprocess import json import pkg_resources -import urlparse -from copy import copy, deepcopy +from copy import deepcopy from bkr.inttest import Process from bkr.inttest.labcontroller import LabControllerTestCase from bkr.server.model import OSMajor from turbogears.database import session +from six.moves import urllib + + _current_dir = os.path.dirname(__file__) _compose_test_dir = pkg_resources.resource_filename('bkr.inttest.labcontroller', 'compose_layout') _git_root_dir = os.path.join(_current_dir, '..', '..', '..', '..', '..') @@ -1175,7 +1177,7 @@ def setUp(self): u"type": u"initrd" } ], - u"kernel_options": u" inst.stage2={}".format(urlparse.urljoin( + u"kernel_options": u" inst.stage2={}".format(urllib.parse.urljoin( self.distro_url, u'RHVH4/RHVH-4.3-20200323.0/compose/RHVH/x86_64/os')), u"kernel_options_post": None, u"ks_meta": u" autopart_type=thinp liveimg=Packages/redhat-virtualization-host-image-update-1.0.0-1.noarch.rpm ks_keyword=inst.ks", @@ -1199,7 +1201,7 @@ def setUp(self): ], u"tree_build_time": u"1584961599", u"urls": [ - u"{}".format(urlparse.urljoin(self.distro_url, u'RHVH4/RHVH-4.3-20200323.0/compose/RHVH/x86_64/os/')) + u"{}".format(urllib.parse.urljoin(self.distro_url, u'RHVH4/RHVH-4.3-20200323.0/compose/RHVH/x86_64/os/')) ], u"variant": u"RHVH" } @@ -1208,7 +1210,7 @@ def _run_import(self, import_args): p = subprocess.Popen(import_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=dict(os.environ.items() + [('PYTHONUNBUFFERED', '1')])) + env=dict(list(os.environ.items()) + [('PYTHONUNBUFFERED', '1')])) stdout, stderr = p.communicate() if p.returncode: raise TreeImportError(import_args, p.returncode, stderr) @@ -1220,7 +1222,7 @@ def dry_run_import_trees(self, additional_import_args): trees, stderr = self._run_import( ['python', _command, '--debug', '--json', '--dry-run'] + additional_import_args) - print stderr + print(stderr) # check logging is working correctly self.assertIn('Dry Run only', stderr) self.assertIn('Attempting to import: ', stderr) @@ -1559,6 +1561,6 @@ def test_rhel8_partner_import(self): self.assertItemsEqual(trees, [self.x86_64_rhel8_partner]) def test_rhvh43_import(self): - trees = self.dry_run_import_trees(['{}'.format(urlparse.urljoin(self.distro_url, + trees = self.dry_run_import_trees(['{}'.format(urllib.parse.urljoin(self.distro_url, 'RHVH4/RHVH-4.3-20200323.0/compose'))]) self.assertItemsEqual(trees, [self.x86_64_rhvh43]) diff --git a/IntegrationTests/src/bkr/inttest/labcontroller/test_provision.py b/IntegrationTests/src/bkr/inttest/labcontroller/test_provision.py index f47a6ab6f..d99b3ec55 100644 --- a/IntegrationTests/src/bkr/inttest/labcontroller/test_provision.py +++ b/IntegrationTests/src/bkr/inttest/labcontroller/test_provision.py @@ -9,7 +9,6 @@ import pkg_resources from turbogears.database import session from unittest import SkipTest, TestCase -from xmlrpclib import _Method from bkr.server.model import PowerType, CommandStatus, System, User, SystemStatus from bkr.labcontroller.config import get_conf from bkr.labcontroller.provision import CommandQueuePoller @@ -17,6 +16,9 @@ from bkr.inttest.assertions import wait_for_condition from bkr.inttest.labcontroller import LabControllerTestCase, processes, daemons_running_externally +from six.moves.xmlrpc_client import _Method + + log = logging.getLogger(__name__) diff --git a/IntegrationTests/src/bkr/inttest/labcontroller/test_proxy.py b/IntegrationTests/src/bkr/inttest/labcontroller/test_proxy.py index ee5b7db49..550a2e4e6 100644 --- a/IntegrationTests/src/bkr/inttest/labcontroller/test_proxy.py +++ b/IntegrationTests/src/bkr/inttest/labcontroller/test_proxy.py @@ -11,13 +11,10 @@ import tempfile import copy from base64 import b64encode -import xmlrpclib import lxml.etree, lxml.html -from urlparse import urljoin import requests import time from nose.plugins.skip import SkipTest -from bkr.common.helpers import total_seconds from bkr.server.model import session, TaskResult, TaskStatus, LogRecipe, \ LogRecipeTask, LogRecipeTaskResult, RecipeTask, RecipeTaskResult from bkr.labcontroller.proxy import ProxyHelper @@ -27,6 +24,12 @@ from bkr.inttest.labcontroller import LabControllerTestCase, processes, \ config_file, daemons_running_externally +from six.moves import reduce +from six.moves import xmlrpc_client +from six.moves.urllib.parse import urljoin + + + class GetRecipeGuestXML(LabControllerTestCase): def setUp(self): @@ -70,7 +73,7 @@ def check_recipe_xml(self, xml): # add more assertions here... def test_xmlrpc_get_my_recipe(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) recipe_xml = s.get_my_recipe({'recipe_id': self.recipe.id}) self.check_recipe_xml(recipe_xml) @@ -82,7 +85,7 @@ def test_xmlrpc_get_recipe_custom_distro(self): # These are optional attributes self.recipe.installation.distro_name = None self.recipe.installation.variant = None - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) recipe_xml = s.get_my_recipe({'recipe_id': self.recipe.id}) self.check_recipe_xml(recipe_xml) @@ -142,28 +145,28 @@ def check_result(self, result_id, result_type, path, score, log): self.assertEquals(result.log, log) def test_xmlrpc_pass(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) result_id = s.task_result(self.recipe.tasks[0].id, u'pass_', u'/random/junk', 123, u'The thing worked') self.check_result(result_id, TaskResult.pass_, u'/random/junk', 123, u'The thing worked') def test_xmlrpc_fail(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) result_id = s.task_result(self.recipe.tasks[0].id, u'fail', u'/random/junk', 456, u'The thing failed') self.check_result(result_id, TaskResult.fail, u'/random/junk', 456, u'The thing failed') def test_xmlrpc_warn(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) result_id = s.task_result(self.recipe.tasks[0].id, u'warn', u'/random/junk', -1, u'The thing broke') self.check_result(result_id, TaskResult.warn, u'/random/junk', -1, u'The thing broke') def test_xmlrpc_panic(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) result_id = s.task_result(self.recipe.tasks[0].id, u'panic', u'/random/junk', 0, u'The thing really broke') self.check_result(result_id, TaskResult.panic, u'/random/junk', 0, @@ -264,8 +267,8 @@ def test_max_results_per_recipe_limit_is_enforced(self): session.expire(self.recipe.tasks[0]) self.assertEqual(len(self.recipe.tasks[0].results), 7500) # Test XMLRPC endpoint - s = xmlrpclib.ServerProxy(self.get_proxy_url()) - with self.assertRaisesRegexp(xmlrpclib.Fault, 'Too many results'): + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) + with self.assertRaisesRegexp(xmlrpc_client.Fault, 'Too many results'): s.task_result(self.recipe.tasks[0].id, 'pass_', '/', 0, 'Should fail') # Test POST endpoint results_url = '%srecipes/%s/tasks/%s/results/' % (self.get_proxy_url(), @@ -291,7 +294,7 @@ def setUp(self): data_setup.mark_recipe_running(self.recipe) def test_xmlrpc_task_start(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.task_start(self.recipe.tasks[0].id) with session.begin(): session.expire_all() @@ -299,7 +302,7 @@ def test_xmlrpc_task_start(self): self.assertEquals(task.status, TaskStatus.running) def test_xmlrpc_task_stop(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.task_stop(self.recipe.tasks[0].id, 'stop') with session.begin(): session.expire_all() @@ -314,11 +317,11 @@ def test_cannot_complete_aborted_task(self): # request should fail. with session.begin(): self.recipe.abort(u'someone ran rhts-abort -t recipe') - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) try: s.task_stop(self.recipe.tasks[0].id, 'stop') self.fail('should raise') - except xmlrpclib.Fault as fault: + except xmlrpc_client.Fault as fault: self.assertIn('Cannot change status for finished task', fault.faultString) with session.begin(): @@ -327,7 +330,7 @@ def test_cannot_complete_aborted_task(self): self.assertEquals(task.status, TaskStatus.aborted) def test_xmlrpc_task_abort(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.task_stop(self.recipe.tasks[0].id, 'abort', 'fooed the bar up') with session.begin(): session.expire_all() @@ -505,7 +508,7 @@ def setUp(self): data_setup.mark_recipe_running(self.recipe) def test_xmlrpc_recipe_abort(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.recipe_stop(self.recipe.id, 'abort', 'fooed the bar up') with session.begin(): session.expire_all() @@ -541,7 +544,7 @@ def setUp(self): data_setup.mark_recipe_running(self.recipe) def test_xmlrpc_extend_watchdog(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) result_id = s.extend_watchdog(self.recipe.tasks[0].id, 600) with session.begin(): session.expire_all() @@ -579,7 +582,7 @@ def setUpClass(cls): def test_clear_netboot(self): with session.begin(): system = data_setup.create_system() - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.clear_netboot(system.fqdn) with session.begin(): self.assertEqual(system.command_queue[0].action, 'clear_netboot') @@ -601,7 +604,7 @@ def setUp(self): data_setup.mark_recipe_waiting(self.recipe) def test_install_start(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.install_start(self.recipe.id) with session.begin(): session.expire_all() @@ -627,7 +630,7 @@ def test_install_start_GET(self): # https://bugzilla.redhat.com/show_bug.cgi?id=954219 def test_install_start_faulty(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) with session.begin(): recipe = data_setup.create_recipe() @@ -663,7 +666,7 @@ def setUp(self): self.assertIsNone(self.recipe.resource.fqdn) def test_install_done(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.install_done(self.recipe.id, 'somefqdn') with session.begin(): session.expire_all() @@ -698,7 +701,7 @@ def setUp(self): data_setup.mark_recipe_installing(self.recipe) def test_install_fail(self): - s = xmlrpclib.Server(self.get_proxy_url()) + s = xmlrpc_client.Server(self.get_proxy_url()) s.install_fail(self.recipe.id) with session.begin(): session.expire_all() @@ -732,7 +735,7 @@ def setUp(self): data_setup.mark_recipe_running(self.recipe, system=self.system) def test_postreboot(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.postreboot(self.recipe.id) with session.begin(): session.expire_all() @@ -763,7 +766,7 @@ def setUp(self): # https://bugzilla.redhat.com/show_bug.cgi?id=1501671 def test_power(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) s.power(self.target_system.fqdn, 'off') with session.begin(): session.expire_all() @@ -866,7 +869,7 @@ def _run_modified_log_url(self, config_arg): new_args.append(tmp_config.name) beaker_proxy_process.args = new_args beaker_proxy_process.start() - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) s.recipe_upload_file(self.recipe.id, '/', 'recipe-log', 10, None, 0, b64encode('a' * 10)) @@ -892,7 +895,7 @@ def test_log_storage_base_url(self): self.assertEquals(proxy.log_storage.base_url, 'https://testingme.com/beaker/logs/') def test_xmlrpc_recipe_log(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) s.recipe_upload_file(self.recipe.id, '/', 'recipe-log', 10, None, 0, b64encode('a' * 10)) local_log_dir = '%s/recipes/%s+/%s/' % (get_conf().get('CACHEPATH'), @@ -919,12 +922,12 @@ def test_xmlrpc_recipe_log_after_finished(self): with session.begin(): data_setup.mark_recipe_complete(self.recipe, only=True) assert self.recipe.is_finished() - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) try: s.recipe_upload_file(self.recipe.id, '/', 'recipe-log', 10, None, 0, b64encode('a' * 10)) self.fail('should raise') - except xmlrpclib.Fault, fault: + except xmlrpc_client.Fault as fault: self.assert_('Cannot register file for finished recipe' in fault.faultString) @@ -969,7 +972,7 @@ def test_PUT_recipe_log_after_finished(self): def test_xmlrpc_task_log(self): with session.begin(): task = self.recipe.tasks[0] - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) s.task_upload_file(task.id, '/', 'task-log', 10, None, 0, b64encode('a' * 10)) local_log_dir = '%s/tasks/%s+/%s/' % (get_conf().get('CACHEPATH'), @@ -997,12 +1000,12 @@ def test_xmlrpc_task_log_after_finished(self): task = self.recipe.tasks[0] task.stop() assert task.is_finished() - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) try: s.task_upload_file(task.id, '/', 'task-log', 10, None, 0, b64encode('a' * 10)) self.fail('should raise') - except xmlrpclib.Fault, fault: + except xmlrpc_client.Fault as fault: self.assert_('Cannot register file for finished task' in fault.faultString) @@ -1051,7 +1054,7 @@ def test_xmlrpc_result_log(self): with session.begin(): self.recipe.tasks[0].pass_(u'', 0, u'Pass') result = self.recipe.tasks[0].results[0] - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) s.result_upload_file(result.id, '/', 'result-log', 10, None, 0, b64encode('a' * 10)) local_log_dir = '%s/results/%s+/%s/' % (get_conf().get('CACHEPATH'), @@ -1080,12 +1083,12 @@ def test_xmlrpc_result_log_after_finished(self): result = self.recipe.tasks[0].results[0] self.recipe.tasks[0].stop() assert self.recipe.tasks[0].is_finished() - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) try: s.result_upload_file(result.id, '/', 'result-log-after-finished', 10, None, 0, b64encode('a' * 10)) self.fail('should raise') - except xmlrpclib.Fault, fault: + except xmlrpc_client.Fault as fault: self.assert_('Cannot register file for finished task' in fault.faultString) @@ -1154,12 +1157,12 @@ def test_PUT_empty_log(self): # https://bugzilla.redhat.com/show_bug.cgi?id=1003454 def test_large_xmlrpc_request_is_rejected(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) try: s.task_upload_file(123, 'debug', '.task_beah_raw', 4096, '', 1024, 'a' * (1024 * 1024 * 10 + 1)) self.fail('should raise') - except xmlrpclib.ProtocolError as e: + except xmlrpc_client.ProtocolError as e: self.assertEquals(e.errcode, 413) # https://bugzilla.redhat.com/show_bug.cgi?id=1003454 @@ -1198,8 +1201,8 @@ def test_max_logs_per_recipe_limit_is_enforced(self): session.expire(result) self.assertEqual(len(result.logs), 3500) # Test XMLRPC endpoint for result logs - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) - with self.assertRaisesRegexp(xmlrpclib.Fault, 'Too many logs'): + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) + with self.assertRaisesRegexp(xmlrpc_client.Fault, 'Too many logs'): s.result_upload_file(result.id, '/', 'result-log', 10, None, 0, b64encode('a' * 10)) # Test POST endpoint for result logs @@ -1209,8 +1212,8 @@ def test_max_logs_per_recipe_limit_is_enforced(self): self.assertEquals(response.status_code, 403) self.assertIn('Too many logs in recipe', response.text) # Test XMLRPC endpoint for task logs - s = xmlrpclib.ServerProxy(self.get_proxy_url(), allow_none=True) - with self.assertRaisesRegexp(xmlrpclib.Fault, 'Too many logs'): + s = xmlrpc_client.ServerProxy(self.get_proxy_url(), allow_none=True) + with self.assertRaisesRegexp(xmlrpc_client.Fault, 'Too many logs'): s.task_upload_file(task.id, '/', 'task-log', 10, None, 0, b64encode('a' * 10)) # Test POST endpoint for task logs @@ -1320,7 +1323,7 @@ def setUp(self): # https://bugzilla.redhat.com/show_bug.cgi?id=978640 def test_can_get_last_installation_info(self): - s = xmlrpclib.ServerProxy(self.get_proxy_url()) + s = xmlrpc_client.ServerProxy(self.get_proxy_url()) installinfo = s.get_installation_for_system(self.recipe.resource.fqdn) self.assertItemsEqual(installinfo['distro_tree_urls'], self.tree_urls) self.assertEqual(installinfo['kernel_url'],