From 39bb2288b680337829bba0a8df91a3728e5077cf Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Fri, 1 Sep 2023 22:59:14 +0300 Subject: [PATCH 01/12] steps: Fix Git step when using shallow and submodules options --- master/buildbot/steps/source/git.py | 2 +- .../test/unit/steps/test_source_git.py | 32 +++++++++++++++++++ .../git-incremental-submodule-shallow.bugfix | 1 + 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 newsfragments/git-incremental-submodule-shallow.bugfix diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index 4fa8ebafb71f..296fb92a6ac4 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -442,7 +442,7 @@ def _fullClone(self, shallowClone=False): cmdArgs = ["submodule", "update", "--init", "--recursive"] if self.remoteSubmodules: cmdArgs.append("--remote") - if self.shallow: + if shallowClone: cmdArgs.extend(["--depth", str(int(shallowClone))]) res = yield self._dovccmd(cmdArgs, shallowClone) diff --git a/master/buildbot/test/unit/steps/test_source_git.py b/master/buildbot/test/unit/steps/test_source_git.py index d0331dbdfda4..dbd7d8d14ad5 100644 --- a/master/buildbot/test/unit/steps/test_source_git.py +++ b/master/buildbot/test/unit/steps/test_source_git.py @@ -1712,6 +1712,38 @@ def test_mode_incremental_branch_ssh_key_2_10(self): 'got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', self.sourceName) return self.run_step() + def test_mode_incremental_no_existing_repo_shallow_submodules(self): + self.setup_step( + self.stepClass(repourl='http://github.com/buildbot/buildbot.git', + mode='incremental', shallow=True, submodules=True)) + + self.expect_commands( + ExpectShell(workdir='wkdir', + command=['git', '--version']) + .stdout('git version 1.7.5') + .exit(0), + ExpectStat(file='wkdir/.buildbot-patched', log_environ=True) + .exit(1), + ExpectListdir(dir='wkdir') + .exit(0), + ExpectShell(workdir='wkdir', + command=['git', 'clone', + 'http://github.com/buildbot/buildbot.git', + '.', '--progress']) + .exit(0), + ExpectShell(workdir='wkdir', + command=['git', 'submodule', 'update', '--init', '--recursive']) + .exit(0), + ExpectShell(workdir='wkdir', + command=['git', 'rev-parse', 'HEAD']) + .stdout('f6ad368298bd941e934a41f3babc827b2aa95a1d') + .exit(0) + ) + self.expect_outcome(result=SUCCESS) + self.expect_property( + 'got_revision', 'f6ad368298bd941e934a41f3babc827b2aa95a1d', self.sourceName) + return self.run_step() + def test_mode_full_fresh(self): self.setup_step( self.stepClass(repourl='http://github.com/buildbot/buildbot.git', diff --git a/newsfragments/git-incremental-submodule-shallow.bugfix b/newsfragments/git-incremental-submodule-shallow.bugfix new file mode 100644 index 000000000000..cc7567a9dc55 --- /dev/null +++ b/newsfragments/git-incremental-submodule-shallow.bugfix @@ -0,0 +1 @@ +Fix incorrect git command line parameters when using ``Git`` source step with ``mode="incremental"``, ``shallow=True``, ``submodules=True`` (regression since Buildbot 3.9) (:issue:`7054`). From 70b24feee6de840413f3e75a3844634efda1c6e0 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Fri, 1 Sep 2023 22:59:15 +0300 Subject: [PATCH 02/12] doc: Remove irrelevant comment about what Buildbot needs from git --- master/docs/manual/configuration/steps/source_git.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/docs/manual/configuration/steps/source_git.rst b/master/docs/manual/configuration/steps/source_git.rst index 3f6fb81d0f54..a2e0570afb4c 100644 --- a/master/docs/manual/configuration/steps/source_git.rst +++ b/master/docs/manual/configuration/steps/source_git.rst @@ -11,7 +11,7 @@ The :bb:step:`Git` build step clones or updates a `Git `_ rep .. note:: - Buildbot supports Git version 1.2.0 or later. Earlier versions (such as the one shipped in Ubuntu 'Dapper') do not support the :command:`git init` command that Buildbot uses. + Buildbot supports Git version 1.2.0 or later. .. code-block:: python From 5340ceff213f05d50369ccf7efbababfed896ed8 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Fri, 1 Sep 2023 22:59:16 +0300 Subject: [PATCH 03/12] doc: Clarify support for shallow option in Git step --- master/buildbot/steps/source/git.py | 2 +- master/buildbot/test/unit/steps/test_source_git.py | 2 +- master/docs/manual/configuration/steps/source_git.rst | 2 +- newsfragments/git-shallow-incremental.doc | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 newsfragments/git-shallow-incremental.doc diff --git a/master/buildbot/steps/source/git.py b/master/buildbot/steps/source/git.py index 296fb92a6ac4..f1d15b66ac81 100644 --- a/master/buildbot/steps/source/git.py +++ b/master/buildbot/steps/source/git.py @@ -114,7 +114,7 @@ def __init__(self, repourl=None, branch='HEAD', mode='incremental', method=None, bbconfig.error("Git: invalid method for mode 'full'.") if self.shallow and (self.mode != 'full' or self.method != 'clobber'): bbconfig.error( - "Git: shallow only possible with mode 'full' and method 'clobber'.") + "Git: in mode 'full' shallow only possible with method 'clobber'.") if not isinstance(self.getDescription, (bool, dict)): bbconfig.error("Git: getDescription must be a boolean or a dict.") diff --git a/master/buildbot/test/unit/steps/test_source_git.py b/master/buildbot/test/unit/steps/test_source_git.py index dbd7d8d14ad5..85a1001c5cab 100644 --- a/master/buildbot/test/unit/steps/test_source_git.py +++ b/master/buildbot/test/unit/steps/test_source_git.py @@ -2366,7 +2366,7 @@ def test_mode_full_copy_ssh_key_2_10(self): def test_mode_full_copy_shallow(self): with self.assertRaisesConfigError( - "shallow only possible with mode 'full' and method 'clobber'"): + "in mode 'full' shallow only possible with method 'clobber'"): self.stepClass(repourl='http://github.com/buildbot/buildbot.git', mode='full', method='copy', shallow=True) diff --git a/master/docs/manual/configuration/steps/source_git.rst b/master/docs/manual/configuration/steps/source_git.rst index a2e0570afb4c..82e633be88bf 100644 --- a/master/docs/manual/configuration/steps/source_git.rst +++ b/master/docs/manual/configuration/steps/source_git.rst @@ -37,7 +37,7 @@ The Git step takes the following arguments: ``shallow`` (optional) Instructs Git to attempt shallow clones (``--depth 1``). The depth defaults to 1 and can be changed by passing an integer instead of ``True``. - This option can be used only in full builds with clobber method. + This option can be used only in incremental builds, or full builds with clobber method. ``reference`` (optional) Use the specified string as a path to a reference repository on the local machine. diff --git a/newsfragments/git-shallow-incremental.doc b/newsfragments/git-shallow-incremental.doc new file mode 100644 index 000000000000..56286093d05c --- /dev/null +++ b/newsfragments/git-shallow-incremental.doc @@ -0,0 +1 @@ +Clarified that ``shallow`` option for the ``Git`` source step is also supported in ``incremental`` mode. From 177fe428312e15e2e1a10e7e85ac06814e362e20 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sat, 2 Sep 2023 01:29:14 +0300 Subject: [PATCH 04/12] github: Run builds on all pull requests --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10bc999969cd..a03d7dcaa40f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,8 +4,6 @@ on: branches: - master pull_request: - branches: - - master permissions: contents: read From 411a7b269e69aac1750e4ea4c556e95414637cc6 Mon Sep 17 00:00:00 2001 From: vibbo Date: Fri, 1 Sep 2023 17:00:30 +0200 Subject: [PATCH 05/12] worker: Improved check of "number" options for create-worker Options that will be converted to numbers have to be valid Python literals. We now check that such number does not start with 0. Fixes #7047 --- newsfragments/create-worker-options-check.bugfix | 1 + worker/buildbot_worker/scripts/runner.py | 2 +- worker/buildbot_worker/test/unit/test_scripts_runner.py | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 newsfragments/create-worker-options-check.bugfix diff --git a/newsfragments/create-worker-options-check.bugfix b/newsfragments/create-worker-options-check.bugfix new file mode 100644 index 000000000000..cdb81b2e539a --- /dev/null +++ b/newsfragments/create-worker-options-check.bugfix @@ -0,0 +1 @@ +Options for `create-worker` that are converted to numbers are now also checked to be valid Python literals. This will prevent creating invalid worker configurations, e.g.: when using option ``--umask=022`` instead of ``--umask=0o022`` or ``--umask=18``. (:issue:`7047`) diff --git a/worker/buildbot_worker/scripts/runner.py b/worker/buildbot_worker/scripts/runner.py index 805913e66525..57cbe7cc00db 100644 --- a/worker/buildbot_worker/scripts/runner.py +++ b/worker/buildbot_worker/scripts/runner.py @@ -217,7 +217,7 @@ def postOptions(self): argument)) for argument in ["log-count", "maxretries", "umask", "numcpus"]: - if not re.match(r'^(0o)?\d+$', self[argument]) and \ + if not re.match(r'^((0o)\d+|0|[1-9]\d*)$', self[argument]) and \ self[argument] != 'None': raise usage.UsageError("{} parameter needs to be a number" " or None".format(argument)) diff --git a/worker/buildbot_worker/test/unit/test_scripts_runner.py b/worker/buildbot_worker/test/unit/test_scripts_runner.py index e781b1413575..654cca1ba9b5 100644 --- a/worker/buildbot_worker/test/unit/test_scripts_runner.py +++ b/worker/buildbot_worker/test/unit/test_scripts_runner.py @@ -244,6 +244,11 @@ def test_inv_umask(self): "umask parameter needs to be a number or None"): self.parse("--umask=X", *self.req_args) + def test_inv_umask2(self): + with self.assertRaisesRegex(usage.UsageError, + "umask parameter needs to be a number or None"): + self.parse("--umask=022", *self.req_args) + def test_inv_allow_shutdown(self): with self.assertRaisesRegex(usage.UsageError, "allow-shutdown needs to be one of 'signal' or 'file'"): From a991611697cafcbd9d37b88b648d85ec3d2e74fd Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sun, 27 Aug 2023 12:39:53 +0300 Subject: [PATCH 06/12] scripts: Adjust autoincrement sequences for primary keys on Postgres When inserting explicit primary key values on Postgres the stored sequence values are not updated and start from zero. This leads to duplicate key errors on insertion later because the sequences return primary key values that already exist. --- master/buildbot/scripts/copydb.py | 22 ++++++++++++++++++++++ newsfragments/copydb-autoincrement.bugfix | 1 + 2 files changed, 23 insertions(+) create mode 100644 newsfragments/copydb-autoincrement.bugfix diff --git a/master/buildbot/scripts/copydb.py b/master/buildbot/scripts/copydb.py index 6d62e2e7edf6..45bc6d70a582 100644 --- a/master/buildbot/scripts/copydb.py +++ b/master/buildbot/scripts/copydb.py @@ -91,11 +91,29 @@ def _copy_single_table(src_db, dst_db, table, table_name, buildset_to_parent_bui written_count = [0] total_count = [0] + autoincrement_foreign_key_column = None + for column_name, column in table.columns.items(): + if not column.foreign_keys and column.primary_key and isinstance(column.type, sa.Integer): + autoincrement_foreign_key_column = column_name + def thd_write(conn): + max_column_id = 0 while True: try: rows = rows_queue.get(timeout=1) if rows is None: + + if autoincrement_foreign_key_column is not None and max_column_id != 0: + if dst_db.pool.engine.dialect.name == 'postgresql': + # Explicitly inserting primary row IDs does not bump the primary key + # sequence on Postgres + seq_name = f"{table_name}_{autoincrement_foreign_key_column}_seq" + transaction = conn.begin() + conn.execute( + f"ALTER SEQUENCE {seq_name} RESTART WITH {max_column_id + 1}" + ) + transaction.commit() + rows_queue.task_done() return @@ -103,6 +121,10 @@ def thd_write(conn): {k: getattr(row, k) for k in column_keys} for row in rows ] + if autoincrement_foreign_key_column is not None: + for row in row_dicts: + max_column_id = max(max_column_id, row[autoincrement_foreign_key_column]) + if table_name == "buildsets": for row_dict in row_dicts: if row_dict["parent_buildid"] is not None: diff --git a/newsfragments/copydb-autoincrement.bugfix b/newsfragments/copydb-autoincrement.bugfix new file mode 100644 index 000000000000..f1f1eced4b3b --- /dev/null +++ b/newsfragments/copydb-autoincrement.bugfix @@ -0,0 +1 @@ +Fix handling of primary key columns on Postgres in the ``copy-db`` script. From 27ed633b8e53af6772f430b7fcf55c23c41adde3 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sun, 27 Aug 2023 12:39:54 +0300 Subject: [PATCH 07/12] scripts: Fix race condition in copying data in the copydb script Missing wait for completion of _copy_single_table lead to the script finishing prematurely. Under certain conditions the write thread may not start and thus data wouldn't be written to the destination database. --- master/buildbot/scripts/copydb.py | 2 +- newsfragments/copydb-tables.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/copydb-tables.bugfix diff --git a/master/buildbot/scripts/copydb.py b/master/buildbot/scripts/copydb.py index 45bc6d70a582..b72ce5d924f2 100644 --- a/master/buildbot/scripts/copydb.py +++ b/master/buildbot/scripts/copydb.py @@ -223,7 +223,7 @@ def _copy_database_with_db(src_db, dst_db, print_log): for table_name in table_names: table = metadata.tables[table_name] - _copy_single_table( + yield _copy_single_table( src_db, dst_db, table, diff --git a/newsfragments/copydb-tables.bugfix b/newsfragments/copydb-tables.bugfix new file mode 100644 index 000000000000..1ae7a77b1319 --- /dev/null +++ b/newsfragments/copydb-tables.bugfix @@ -0,0 +1 @@ +Fixed a race condition in the ``copy-db`` script which sometimes lead to no data being copied. From 40bff89be53061ecadbf739a81fd9b0687700ad0 Mon Sep 17 00:00:00 2001 From: vibbo Date: Tue, 22 Aug 2023 14:48:06 +0200 Subject: [PATCH 08/12] worker: Log error in remote_getWorkerInfo instead of crashing Worker lists and reads all file in WORKER/info folder in functin remote_getWorkerInfo. To prevent it from crashing when it encounteres file that can not be decoded we now only report this error to log and continue. Fixes #3585 Fixes #4758 Fixes #6932 --- newsfragments/get-worker-info.bugfix | 1 + worker/buildbot_worker/base.py | 8 ++++- worker/buildbot_worker/test/unit/test_bot.py | 35 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 newsfragments/get-worker-info.bugfix diff --git a/newsfragments/get-worker-info.bugfix b/newsfragments/get-worker-info.bugfix new file mode 100644 index 000000000000..54fc902996b2 --- /dev/null +++ b/newsfragments/get-worker-info.bugfix @@ -0,0 +1 @@ +Fix worker not connecting error when there are files in WORKER/info folder that can not be decoded. (:issue:`3585`) (:issue:`4758`) (:issue:`6932`) diff --git a/worker/buildbot_worker/base.py b/worker/buildbot_worker/base.py index cb80c4018093..21470e061e32 100644 --- a/worker/buildbot_worker/base.py +++ b/worker/buildbot_worker/base.py @@ -25,12 +25,14 @@ from twisted.application import service from twisted.internet import defer from twisted.internet import reactor +from twisted.python import failure from twisted.python import log from twisted.spread import pb import buildbot_worker from buildbot_worker.commands import base from buildbot_worker.commands import registry +from buildbot_worker.compat import bytes2unicode from buildbot_worker.util import buffer_manager from buildbot_worker.util import lineboundaries @@ -218,7 +220,11 @@ def remote_getWorkerInfo(self): filename = os.path.join(basedir, f) if os.path.isfile(filename): with open(filename, "r") as fin: - files[f] = fin.read() + try: + files[f] = bytes2unicode(fin.read()) + except UnicodeDecodeError: + log.err(failure.Failure(), + 'error while reading file: %s' % (filename)) self._read_os_release(self.os_release_file, files) diff --git a/worker/buildbot_worker/test/unit/test_bot.py b/worker/buildbot_worker/test/unit/test_bot.py index 03cff1c22f88..bf7f739f7a19 100644 --- a/worker/buildbot_worker/test/unit/test_bot.py +++ b/worker/buildbot_worker/test/unit/test_bot.py @@ -127,6 +127,41 @@ def test_getWorkerInfo_nodir(self): ['environ', 'system', 'numcpus', 'basedir', 'worker_commands', 'version', 'delete_leftover_dirs'])) + @defer.inlineCallbacks + def test_getWorkerInfo_decode_error(self): + infodir = os.path.join(self.basedir, "info") + os.makedirs(infodir) + with open(os.path.join(infodir, "admin"), "w") as f: + f.write("testy!") + with open(os.path.join(infodir, "foo"), "w") as f: + f.write("bar") + with open(os.path.join(infodir, "environ"), "w") as f: + f.write("something else") + # This will not be part of worker info + with open(os.path.join(infodir, "binary"), "wb") as f: + f.write(b"\x90") + + # patch the log.err, otherwise trial will think something *actually* + # failed + self.patch(log, "err", lambda f, x: None) + + info = yield self.bot.callRemote("getWorkerInfo") + + # remove any os_ fields as they are dependent on the test environment + info = {k: v for k, v in info.items() if not k.startswith("os_")} + + self.assertEqual(info, { + "admin": 'testy!', + "foo": 'bar', + "environ": os.environ, + "system": os.name, + "basedir": self.basedir, + "worker_commands": self.real_bot.remote_getCommands(), + "version": self.real_bot.remote_getVersion(), + "numcpus": multiprocessing.cpu_count(), + "delete_leftover_dirs": False + }) + def test_shutdown(self): d1 = defer.Deferred() self.patch(reactor, "stop", lambda: d1.callback(None)) From d764e8157674a3d53cbf7949183f9eae75fdb1d9 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sat, 2 Sep 2023 18:29:47 +0300 Subject: [PATCH 09/12] docs: Remove note about Python 2.7 no longer supported --- master/docs/conf.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/master/docs/conf.py b/master/docs/conf.py index bcac587eefa8..e5bf1b893409 100755 --- a/master/docs/conf.py +++ b/master/docs/conf.py @@ -87,10 +87,7 @@ # The full version, including alpha/beta/rc tags. release = version -# add a loud note about python 2 -rst_prolog = textwrap.dedent("""\ -.. caution:: Buildbot no longer supports Python 2.7 on the Buildbot master. -""") +rst_prolog = "" # add a loud note for anyone looking at the latest docs if release == 'latest': From 338bcaab09d03c60845cca057e3fb2830e4bb0bb Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sat, 2 Sep 2023 18:25:09 +0300 Subject: [PATCH 10/12] Release notes for 3.9.1 --- master/docs/relnotes/index.rst | 20 +++++++++++++++++++ newsfragments/copydb-autoincrement.bugfix | 1 - newsfragments/copydb-tables.bugfix | 1 - .../create-worker-options-check.bugfix | 1 - newsfragments/get-worker-info.bugfix | 1 - .../git-incremental-submodule-shallow.bugfix | 1 - newsfragments/git-shallow-incremental.doc | 1 - 7 files changed, 20 insertions(+), 6 deletions(-) delete mode 100644 newsfragments/copydb-autoincrement.bugfix delete mode 100644 newsfragments/copydb-tables.bugfix delete mode 100644 newsfragments/create-worker-options-check.bugfix delete mode 100644 newsfragments/get-worker-info.bugfix delete mode 100644 newsfragments/git-incremental-submodule-shallow.bugfix delete mode 100644 newsfragments/git-shallow-incremental.doc diff --git a/master/docs/relnotes/index.rst b/master/docs/relnotes/index.rst index 4d475cfc994b..5f561fdacc60 100644 --- a/master/docs/relnotes/index.rst +++ b/master/docs/relnotes/index.rst @@ -8,6 +8,26 @@ Release Notes .. towncrier release notes start + +Buildbot ``3.9.1`` ( ``2023-09-02`` ) +===================================== + +Bug fixes +--------- + +- Fixed handling of primary key columns on Postgres in the ``copy-db`` script. +- Fixed a race condition in the ``copy-db`` script which sometimes lead to no data being copied. +- Options for `create-worker` that are converted to numbers are now also checked to be valid Python literals. + This will prevent creating invalid worker configurations, e.g.: when using option ``--umask=022`` instead of ``--umask=0o022`` or ``--umask=18``. (:issue:`7047`) +- Fixed worker not connecting error when there are files in WORKER/info folder that can not be decoded. (:issue:`3585`) (:issue:`4758`) (:issue:`6932`) +- Fixed incorrect git command line parameters when using ``Git`` source step with ``mode="incremental"``, ``shallow=True``, ``submodules=True`` (regression since Buildbot 3.9.0) (:issue:`7054`). + +Improved Documentation +---------------------- + +- Clarified that ``shallow`` option for the ``Git`` source step is also supported in ``incremental`` mode. + + Buildbot ``3.9.0`` ( ``2023-08-16`` ) ===================================== diff --git a/newsfragments/copydb-autoincrement.bugfix b/newsfragments/copydb-autoincrement.bugfix deleted file mode 100644 index f1f1eced4b3b..000000000000 --- a/newsfragments/copydb-autoincrement.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix handling of primary key columns on Postgres in the ``copy-db`` script. diff --git a/newsfragments/copydb-tables.bugfix b/newsfragments/copydb-tables.bugfix deleted file mode 100644 index 1ae7a77b1319..000000000000 --- a/newsfragments/copydb-tables.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fixed a race condition in the ``copy-db`` script which sometimes lead to no data being copied. diff --git a/newsfragments/create-worker-options-check.bugfix b/newsfragments/create-worker-options-check.bugfix deleted file mode 100644 index cdb81b2e539a..000000000000 --- a/newsfragments/create-worker-options-check.bugfix +++ /dev/null @@ -1 +0,0 @@ -Options for `create-worker` that are converted to numbers are now also checked to be valid Python literals. This will prevent creating invalid worker configurations, e.g.: when using option ``--umask=022`` instead of ``--umask=0o022`` or ``--umask=18``. (:issue:`7047`) diff --git a/newsfragments/get-worker-info.bugfix b/newsfragments/get-worker-info.bugfix deleted file mode 100644 index 54fc902996b2..000000000000 --- a/newsfragments/get-worker-info.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix worker not connecting error when there are files in WORKER/info folder that can not be decoded. (:issue:`3585`) (:issue:`4758`) (:issue:`6932`) diff --git a/newsfragments/git-incremental-submodule-shallow.bugfix b/newsfragments/git-incremental-submodule-shallow.bugfix deleted file mode 100644 index cc7567a9dc55..000000000000 --- a/newsfragments/git-incremental-submodule-shallow.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect git command line parameters when using ``Git`` source step with ``mode="incremental"``, ``shallow=True``, ``submodules=True`` (regression since Buildbot 3.9) (:issue:`7054`). diff --git a/newsfragments/git-shallow-incremental.doc b/newsfragments/git-shallow-incremental.doc deleted file mode 100644 index 56286093d05c..000000000000 --- a/newsfragments/git-shallow-incremental.doc +++ /dev/null @@ -1 +0,0 @@ -Clarified that ``shallow`` option for the ``Git`` source step is also supported in ``incremental`` mode. From 7b33590abaceb9bf28cdff890d641aa916fb1205 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sat, 2 Sep 2023 19:28:15 +0300 Subject: [PATCH 11/12] master: Pin Twisted to 22.10 or older This works around the following exception: Unhandled error in Deferred: Traceback (most recent call last): File "/.../site-packages/buildbot/scripts/create_master.py", line 84, in createDB master = BuildMaster(config['basedir']) File "/.../site-packages/buildbot/master.py", line 102, in __init__ self._services_d = self.create_child_services() File "/.../site-packages/twisted/internet/defer.py", line 2245, in unwindGenerator return _cancellableInlineCallbacks(gen) File "/.../site-packages/twisted/internet/defer.py", line 2157, in _cancellableInlineCallbacks _inlineCallbacks(None, gen, status, _copy_context()) --- --- File "/.../site-packages/twisted/internet/defer.py", line 1997, in _inlineCallbacks result = context.run(gen.send, result) File "/.../site-packages/buildbot/master.py", line 188, in create_child_services self.www = wwwservice.WWWService() File "/.../site-packages/buildbot/www/service.py", line 196, in __init__ self.apps = get_plugins('www', None, load_now=True) File "/.../site-packages/buildbot/plugins/db.py", line 356, in get_plugins return _DB.add_namespace(namespace, interface, check_extras, load_now) File "/.../site-packages/buildbot/plugins/db.py", line 306, in add_namespace tempo.load() File "/.../site-packages/buildbot/plugins/db.py", line 242, in load self._tree.load() File "/.../site-packages/buildbot/plugins/db.py", line 112, in load child.load() File "/.../site-packages/buildbot/plugins/db.py", line 45, in load self._value = self._loader(self._entry) File "/.../site-packages/buildbot/plugins/db.py", line 214, in _load_entry raise PluginDBError('Requirements are not satisfied ' buildbot.errors.PluginDBError: Requirements are not satisfied for buildbot.www:base: The 'zope-interface>=5' distribution was not found and is required by Twisted --- master/setup.py | 2 +- newsfragments/pin-twisted.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/pin-twisted.bugfix diff --git a/master/setup.py b/master/setup.py index a9e358feedd2..4e328addbde7 100755 --- a/master/setup.py +++ b/master/setup.py @@ -488,7 +488,7 @@ def define_plugin_entries(groups): if parse_version(pip_dist.version) < parse_version('1.4'): raise RuntimeError(VERSION_MSG) -twisted_ver = ">= 18.7.0" +twisted_ver = ">= 18.7.0, <=22.10.0" bundle_version = version.split("-")[0] diff --git a/newsfragments/pin-twisted.bugfix b/newsfragments/pin-twisted.bugfix new file mode 100644 index 000000000000..9938110d1850 --- /dev/null +++ b/newsfragments/pin-twisted.bugfix @@ -0,0 +1 @@ +Work around requirements parsing error for the Twisted dependency by pinning Twisted to 22.10 or older. From 1874c2faa0831b38acf6d35062720d68f33d3b41 Mon Sep 17 00:00:00 2001 From: Povilas Kanapickas Date: Sat, 2 Sep 2023 22:02:20 +0300 Subject: [PATCH 12/12] Release notes for 3.9.2 --- master/docs/relnotes/index.rst | 10 ++++++++++ newsfragments/pin-twisted.bugfix | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) delete mode 100644 newsfragments/pin-twisted.bugfix diff --git a/master/docs/relnotes/index.rst b/master/docs/relnotes/index.rst index 5f561fdacc60..154c17beafa3 100644 --- a/master/docs/relnotes/index.rst +++ b/master/docs/relnotes/index.rst @@ -9,6 +9,16 @@ Release Notes .. towncrier release notes start +Buildbot ``3.9.2`` ( ``2023-09-02`` ) +===================================== + +Bug fixes +--------- + +- Work around requirements parsing error for the Twisted dependency by pinning Twisted to 22.10 or older. + This fixes buildbot crash on startup when newest Twisted is installed. + + Buildbot ``3.9.1`` ( ``2023-09-02`` ) ===================================== diff --git a/newsfragments/pin-twisted.bugfix b/newsfragments/pin-twisted.bugfix deleted file mode 100644 index 9938110d1850..000000000000 --- a/newsfragments/pin-twisted.bugfix +++ /dev/null @@ -1 +0,0 @@ -Work around requirements parsing error for the Twisted dependency by pinning Twisted to 22.10 or older.