From d2b6a9ae11d52e6f31f4c205cda6e076b396ca36 Mon Sep 17 00:00:00 2001 From: Faustin Lammler Date: Tue, 31 Oct 2023 17:03:59 +0100 Subject: [PATCH] black formatting --- master-libvirt/master.cfg | 369 ++++++++++++++++++++++++-------------- 1 file changed, 234 insertions(+), 135 deletions(-) diff --git a/master-libvirt/master.cfg b/master-libvirt/master.cfg index b4b58cc0..31a14433 100644 --- a/master-libvirt/master.cfg +++ b/master-libvirt/master.cfg @@ -16,7 +16,7 @@ import sys sys.setrecursionlimit(10000) -sys.path.insert(0, '/srv/buildbot/master') +sys.path.insert(0, "/srv/buildbot/master") from utils import * @@ -26,69 +26,97 @@ c = BuildmasterConfig = {} # Load the slave, database passwords and 3rd-party tokens from an external private file, so # that the rest of the configuration can be public. -config = { "private": { } } -exec(open("../master-private.cfg").read(), config, { }) +config = {"private": {}} +exec(open("../master-private.cfg").read(), config, {}) ####### PROJECT IDENTITY # the 'title' string will appear at the top of this buildbot installation's # home pages (linked to the 'titleURL'). -c['title'] = os.getenv('TITLE', default="MariaDB CI") -c['titleURL'] = os.getenv('TITLE_URL', default="https://github.com/MariaDB/server") +c["title"] = os.getenv("TITLE", default="MariaDB CI") +c["titleURL"] = os.getenv("TITLE_URL", default="https://github.com/MariaDB/server") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server is visible. This typically uses the port number set in # the 'www' entry below, but with an externally-visible host name which the # buildbot cannot figure out without some help. -c['buildbotURL'] = os.getenv('BUILDMASTER_URL', default="https://buildbot.mariadb.org/") +c["buildbotURL"] = os.getenv("BUILDMASTER_URL", default="https://buildbot.mariadb.org/") # 'protocols' contains information about protocols which master will use for # communicating with workers. You must define at least 'port' option that workers # could connect to your master with this protocol. # 'port' must match the value configured into the workers (with their # --master option) -port = int(os.getenv('PORT', default="9990")) -c['protocols'] = {'pb': {'port': port}} +port = int(os.getenv("PORT", default="9990")) +c["protocols"] = {"pb": {"port": port}} ####### DB URL -c['db'] = { +c["db"] = { # This specifies what database buildbot uses to store its state. - 'db_url' : config["private"]["db_url"] + "db_url": config["private"]["db_url"] } ####### Disable net usage reports from being sent to buildbot.net -c['buildbotNetUsageData'] = None +c["buildbotNetUsageData"] = None + ####### UTILS def downloadRpms(): return ShellCommand( - name="fetch_rpms", - description="fetching rpms", - descriptionDone="fetching rpms...done", - haltOnFailure=True, - command=["sh", "-xc", util.Interpolate(""" - wget -r -np -nH --cut-dirs=2 -A *.rpm """ + os.getenv('ARTIFACTS_URL', default='https://ci.mariadb.org') + """/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/rpms/" -P . -""")]) + name="fetch_rpms", + description="fetching rpms", + descriptionDone="fetching rpms...done", + haltOnFailure=True, + command=[ + "sh", + "-xc", + util.Interpolate( + """ + wget -r -np -nH --cut-dirs=2 -A *.rpm """ + + os.getenv("ARTIFACTS_URL", default="https://ci.mariadb.org") + + """/%(prop:tarbuildnum)s/%(prop:parentbuildername)s/rpms/" -P . +""" + ), + ], + ) def getRpmUpgradeStep(): - return Test( + return Test( name="upgrade", haltOnFailure=True, description=["testing", "upgrade"], descriptionDone=["test", "upgrade"], - env=envFromProperties(['test_mode', 'test_type', 'branch', 'major_version', 'arch', 'systemdCapability', 'is_main_tree', 'version_name', 'BB_CI']), - command=['./rpm-upgrade.sh']) + env=envFromProperties( + [ + "test_mode", + "test_type", + "branch", + "major_version", + "arch", + "systemdCapability", + "is_main_tree", + "version_name", + "BB_CI", + ] + ), + command=["./rpm-upgrade.sh"], + ) + def getRpmInstallStep(): - return Test( + return Test( name="install", haltOnFailure=True, description=["testing", "install"], descriptionDone=["test", "install"], - env=envFromProperties(['branch', 'master_branch', 'systemdCapability', 'arch', 'BB_CI']), - command=['./rpm-install.sh']) + env=envFromProperties( + ["branch", "master_branch", "systemdCapability", "arch", "BB_CI"] + ), + command=["./rpm-install.sh"], + ) + def getDebGaleraStep(port): def if_run_galera_test(step): @@ -97,17 +125,19 @@ def getDebGaleraStep(port): return True if not branch_is_10_1_or_later(step): return False - if step.getProperty("branch") == 'bb-10.2-compatibility': + if step.getProperty("branch") == "bb-10.2-compatibility": return False - if sst_mode == 'xtrabackup-v2': + if sst_mode == "xtrabackup-v2": if branch_is_10_3_or_later(step): - return False # because of "The redo log was created with MariaDB 10.3.6" + return ( + False # because of "The redo log was created with MariaDB 10.3.6" + ) if branch_is_10_2_or_later(step): - return False # because of MDEV-12289, this might be fixable - if arch not in ['x86', 'i386', 'amd64', 'x86_64']: - return False # no xtrabackup for other architectures - if arch in ['x86', 'i386'] and version_name in ['artful','stretch']: - return False # no 32-bit debs for ubuntu stable and debian 9 + return False # because of MDEV-12289, this might be fixable + if arch not in ["x86", "i386", "amd64", "x86_64"]: + return False # no xtrabackup for other architectures + if arch in ["x86", "i386"] and version_name in ["artful", "stretch"]: + return False # no 32-bit debs for ubuntu stable and debian 9 return True return Test( @@ -118,34 +148,55 @@ def getDebGaleraStep(port): timeout=300, lazylogfiles=True, logfiles={ - "daemon": "/home/buildbot/logs/daemon.log", - "syslog": "/home/buildbot/logs/syslog", - "node1": "/home/buildbot/logs/node1.err", - "node2": "/home/buildbot/logs/node2.err", - "node3": "/home/buildbot/logs/node3.err", - "node1.mariabackup.prepare": "/home/buildbot/logs/node1.mariabackup.prepare.log", - "node2.mariabackup.prepare": "/home/buildbot/logs/node2.mariabackup.prepare.log", - "node3.mariabackup.prepare": "/home/buildbot/logs/node3.mariabackup.prepare.log", - "node1.mariabackup.move": "/home/buildbot/logs/node1.mariabackup.move.log", - "node2.mariabackup.move": "/home/buildbot/logs/node2.mariabackup.move.log", - "node3.mariabackup.move": "/home/buildbot/logs/node3.mariabackup.move.log", - "node1.mariabackup.backup": "/home/buildbot/logs/node1.mariabackup.backup.log", - "node2.mariabackup.backup": "/home/buildbot/logs/node2.mariabackup.backup.log", - "node4.mariabackup.backup": "/home/buildbot/logs/node4.mariabackup.backup.log" + "daemon": "/home/buildbot/logs/daemon.log", + "syslog": "/home/buildbot/logs/syslog", + "node1": "/home/buildbot/logs/node1.err", + "node2": "/home/buildbot/logs/node2.err", + "node3": "/home/buildbot/logs/node3.err", + "node1.mariabackup.prepare": "/home/buildbot/logs/node1.mariabackup.prepare.log", + "node2.mariabackup.prepare": "/home/buildbot/logs/node2.mariabackup.prepare.log", + "node3.mariabackup.prepare": "/home/buildbot/logs/node3.mariabackup.prepare.log", + "node1.mariabackup.move": "/home/buildbot/logs/node1.mariabackup.move.log", + "node2.mariabackup.move": "/home/buildbot/logs/node2.mariabackup.move.log", + "node3.mariabackup.move": "/home/buildbot/logs/node3.mariabackup.move.log", + "node1.mariabackup.backup": "/home/buildbot/logs/node1.mariabackup.backup.log", + "node2.mariabackup.backup": "/home/buildbot/logs/node2.mariabackup.backup.log", + "node4.mariabackup.backup": "/home/buildbot/logs/node4.mariabackup.backup.log", }, doStepIf=if_run_galera_test, - env=envFromProperties(['version_name', 'master_branch', 'sst_mode', 'arch', 'BB_CI']), - command=["./deb-galera.sh"] + env=envFromProperties( + ["version_name", "master_branch", "sst_mode", "arch", "BB_CI"] + ), + command=["./deb-galera.sh"], ) + def getDebUpgradeStep(): return Test( name="upgrade", haltOnFailure=True, description=["testing", "upgrade"], descriptionDone=["test", "upgrade"], - env=envFromProperties(['test_mode', 'test_type', 'branch', 'master_branch', 'arch', 'dist_name', 'version_name', 'mariadb_version', 'major_version', 'systemdCapability', 'needsGalera', 'parentbuildername', 'BB_CI']), - command=['./deb-upgrade.sh']) + env=envFromProperties( + [ + "test_mode", + "test_type", + "branch", + "master_branch", + "arch", + "dist_name", + "version_name", + "mariadb_version", + "major_version", + "systemdCapability", + "needsGalera", + "parentbuildername", + "BB_CI", + ] + ), + command=["./deb-upgrade.sh"], + ) + def getDebInstallStep(): return Test( @@ -153,38 +204,63 @@ def getDebInstallStep(): haltOnFailure=True, description=["testing", "install"], descriptionDone=["test", "install"], - env=envFromProperties(['test_mode', 'test_type', 'branch', 'master_branch', 'arch', 'dist_name', 'version_name', 'mariadb_version', 'major_version', 'systemdCapability', 'needsGalera', 'parentbuildername', 'BB_CI']), - command=['./deb-install.sh']) + env=envFromProperties( + [ + "test_mode", + "test_type", + "branch", + "master_branch", + "arch", + "dist_name", + "version_name", + "mariadb_version", + "major_version", + "systemdCapability", + "needsGalera", + "parentbuildername", + "BB_CI", + ] + ), + command=["./deb-install.sh"], + ) + def getMajorVersionStep(): - return steps.SetPropertyFromCommand(name="major_version", property="major_version", command=util.Interpolate("sh -c \"echo %(prop:mariadb_version)s | sed -e \"s/^mariadb-//\" | sed \"s/.[^.]*$//\"\"")) + return steps.SetPropertyFromCommand( + name="major_version", + property="major_version", + command=util.Interpolate( + 'sh -c "echo %(prop:mariadb_version)s | sed -e "s/^mariadb-//" | sed "s/.[^.]*$//""' + ), + ) + # FACTORY ## f_deb_install f_deb_install = util.BuildFactory() -f_deb_install.addStep(getScript('deb-install.sh')) +f_deb_install.addStep(getScript("deb-install.sh")) f_deb_install.addStep(getDebInstallStep()) -f_deb_install.addStep(getScript('deb-galera.sh')) +f_deb_install.addStep(getScript("deb-galera.sh")) f_deb_install.addStep(getDebGaleraStep("2223")) ## f_deb_upgrade f_deb_upgrade = util.BuildFactory() f_deb_upgrade.addStep(getMajorVersionStep()) -f_deb_upgrade.addStep(getScript('deb-upgrade.sh')) +f_deb_upgrade.addStep(getScript("deb-upgrade.sh")) f_deb_upgrade.addStep(getDebUpgradeStep()) ## f_rpm_install f_rpm_install = util.BuildFactory() f_rpm_install.addStep(downloadRpms()) -f_rpm_install.addStep(getScript('rpm-install.sh')) +f_rpm_install.addStep(getScript("rpm-install.sh")) f_rpm_install.addStep(getRpmInstallStep()) ## f_rpm_upgrade f_rpm_upgrade = util.BuildFactory() f_rpm_upgrade.addStep(getMajorVersionStep()) f_rpm_upgrade.addStep(downloadRpms()) -f_rpm_upgrade.addStep(getScript('rpm-upgrade.sh')) +f_rpm_upgrade.addStep(getScript("rpm-upgrade.sh")) f_rpm_upgrade.addStep(getRpmUpgradeStep()) ####### WORKERS and BUILDERS @@ -192,105 +268,128 @@ f_rpm_upgrade.addStep(getRpmUpgradeStep()) # The 'workers' list defines the set of recognized workers. Each element is # a Worker object, specifying a unique worker name and password. The same # worker name and password must be configured on the worker. -c['workers'] = [] -c['builders'] = [] +c["workers"] = [] +c["builders"] = [] # Add the workers and builds based on the configured install builders (see constants.py) for builder_name in builders_install: # Parse builder name - platform, os_name, os_version, builder_type = builder_name.split('-')[:4] - - assert builder_type in ['rpm', 'deb'] + platform, os_name, os_version, builder_type = builder_name.split("-")[:4] - os_info_name = os_name + '-' + os_version + assert builder_type in ["rpm", "deb"] - libvirt_worker_name = config['private']['libvirt_workers'][platform][0] + '-' + os_name + '-' + os_version + '-' + platform - connection_url = config['private']['libvirt_workers'][platform][1] - image_path = '/var/libvirt/images/' + libvirt_worker_name + os_info_name = os_name + "-" + os_version - c['workers'].append(worker.LibVirtWorker(libvirt_worker_name, - config["private"]["worker_pass"]["libvirt"], - util.Connection(connection_url), - image_path, build_wait_timeout=0, max_builds=1)) + libvirt_worker_name = ( + config["private"]["libvirt_workers"][platform][0] + + "-" + + os_name + + "-" + + os_version + + "-" + + platform + ) + connection_url = config["private"]["libvirt_workers"][platform][1] + image_path = "/var/libvirt/images/" + libvirt_worker_name + + c["workers"].append( + worker.LibVirtWorker( + libvirt_worker_name, + config["private"]["worker_pass"]["libvirt"], + util.Connection(connection_url), + image_path, + build_wait_timeout=0, + max_builds=1, + ) + ) - if builder_type == 'deb': + if builder_type == "deb": factory_install = f_deb_install factory_upgrade = f_deb_upgrade build_arch = platform - elif builder_type == 'rpm': + elif builder_type == "rpm": factory_install = f_rpm_install factory_upgrade = f_rpm_upgrade - build_arch = os_name + os_version + '-' + platform + build_arch = os_name + os_version + "-" + platform - c['builders'].append( - util.BuilderConfig(name=builder_name, + c["builders"].append( + util.BuilderConfig( + name=builder_name, workernames=libvirt_worker_name, - tags=[os_name, builder_type, 'install', 'kvm'], + tags=[os_name, builder_type, "install", "kvm"], collapseRequests=True, nextBuild=nextBuild, canStartBuild=canStartBuild, properties={ - 'systemdCapability': 'yes', - 'needsGalera': 'yes', - 'dist_name': os_name, - 'version_name': os_info[os_info_name]['version_name'], - 'arch': build_arch, - 'BB_CI': True, - }, - factory=factory_install)) + "systemdCapability": "yes", + "needsGalera": "yes", + "dist_name": os_name, + "version_name": os_info[os_info_name]["version_name"], + "arch": build_arch, + "BB_CI": True, + }, + factory=factory_install, + ) + ) # Add major upgrade builder - major_upgrade_name = '-'.join(builder_name.split('-')[:5]) + '-major-upgrade' - - c['builders'].append( - util.BuilderConfig(name=major_upgrade_name, - workernames=libvirt_worker_name, - tags=[os_name, builder_type, 'upgrade', 'kvm'], - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - properties={ - 'systemdCapability': 'yes', - 'needsGalera': 'yes', - 'dist_name': os_name, - 'version_name': os_info[os_info_name]['version_name'], - 'arch': build_arch, - 'test_mode': 'server', - 'test_type': 'major', - 'BB_CI': True, - }, - factory=factory_upgrade)) + major_upgrade_name = "-".join(builder_name.split("-")[:5]) + "-major-upgrade" + + c["builders"].append( + util.BuilderConfig( + name=major_upgrade_name, + workernames=libvirt_worker_name, + tags=[os_name, builder_type, "upgrade", "kvm"], + collapseRequests=True, + nextBuild=nextBuild, + canStartBuild=canStartBuild, + properties={ + "systemdCapability": "yes", + "needsGalera": "yes", + "dist_name": os_name, + "version_name": os_info[os_info_name]["version_name"], + "arch": build_arch, + "test_mode": "server", + "test_type": "major", + "BB_CI": True, + }, + factory=factory_upgrade, + ) + ) # Add minor upgrade builder - minor_upgrade_name = '-'.join(builder_name.split('-')[:5]) + '-minor-upgrade' - - c['builders'].append( - util.BuilderConfig(name=minor_upgrade_name, - workernames=libvirt_worker_name, - tags=[os_name, builder_type, 'upgrade', 'kvm'], - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - properties={ - 'systemdCapability': 'yes', - 'needsGalera': 'yes', - 'dist_name': os_name, - 'version_name': os_info[os_info_name]['version_name'], - 'arch': build_arch, - 'test_mode': 'server', - 'test_type': "minor", - 'BB_CI': True, - }, - factory=factory_upgrade)) - -c['logEncoding'] = 'utf-8' - -c['multiMaster'] = True - -c['mq'] = { # Need to enable multimaster aware mq. Wamp is the only option for now. - 'type' : 'wamp', - 'router_url': os.getenv('MQ_ROUTER_URL', default='ws://localhost:8085/ws'), - 'realm': 'realm1', + minor_upgrade_name = "-".join(builder_name.split("-")[:5]) + "-minor-upgrade" + + c["builders"].append( + util.BuilderConfig( + name=minor_upgrade_name, + workernames=libvirt_worker_name, + tags=[os_name, builder_type, "upgrade", "kvm"], + collapseRequests=True, + nextBuild=nextBuild, + canStartBuild=canStartBuild, + properties={ + "systemdCapability": "yes", + "needsGalera": "yes", + "dist_name": os_name, + "version_name": os_info[os_info_name]["version_name"], + "arch": build_arch, + "test_mode": "server", + "test_type": "minor", + "BB_CI": True, + }, + factory=factory_upgrade, + ) + ) + +c["logEncoding"] = "utf-8" + +c["multiMaster"] = True + +c["mq"] = { # Need to enable multimaster aware mq. Wamp is the only option for now. + "type": "wamp", + "router_url": os.getenv("MQ_ROUTER_URL", default="ws://localhost:8085/ws"), + "realm": "realm1", # valid are: none, critical, error, warn, info, debug, trace - 'wamp_debug_level' : 'info' + "wamp_debug_level": "info", }