dump: add a --verbose flag

For this to work, an optional callback parameter was added to the dump()
methods. The cli passes a callback function to get updates on each new
build's dump phase and that is printed to the screen.

src/gentoo_build_publisher/build_publisher.py

@@ -37,9 +37,11 @@
     Build,
     Change,
     ChangeState,
+    DumpCallback,
     GBPMetadata,
     Package,
     PackageMetadata,
+    default_dump_callback,
 )
 from gentoo_build_publisher.utils.time import utctime
 
@@ -245,23 +247,29 @@ def latest_build(self, machine: str, completed: bool = False) -> BuildRecord | N
         """Return the latest completed build for the given machine name"""
         return self.repo.build_records.latest(machine, completed)
 
-    def dump(self, builds: Iterable[Build], outfile: IO[bytes]) -> None:
+    def dump(
+        self,
+        builds: Iterable[Build],
+        outfile: IO[bytes],
+        *,
+        callback: DumpCallback = default_dump_callback,
+    ) -> None:
         """Dump the given builds to the given outfile"""
         builds = list(builds)
         builds.sort(key=lambda build: (build.machine, build.build_id))
 
         with tar.open(fileobj=outfile, mode="w") as tarfile:
             # first dump storage
             with tempfile.TemporaryFile(mode="w+b") as tmp:
-                self.storage.dump(builds, tmp)
+                self.storage.dump(builds, tmp, callback=callback)
                 tmp.seek(0)
                 tarinfo = tarfile.gettarinfo(arcname="storage.tar", fileobj=tmp)
                 tarfile.addfile(tarinfo, tmp)
 
             # then dump records
             with tempfile.SpooledTemporaryFile(mode="w+b") as tmp:
                 records = [self.repo.build_records.get(build) for build in builds]
-                self.repo.build_records.dump(records, tmp)
+                self.repo.build_records.dump(records, tmp, callback=callback)
                 tmp.seek(0)
                 tarinfo = tarfile.gettarinfo(arcname="records.json", fileobj=tmp)
                 tarfile.addfile(tarinfo, tmp)

src/gentoo_build_publisher/cli/dump.py

@@ -8,6 +8,7 @@
 
 from gentoo_build_publisher import publisher
 from gentoo_build_publisher.records import BuildRecord
+from gentoo_build_publisher.types import Build, DumpPhase
 
 HELP = "Dump builds to a file"
 
@@ -24,14 +25,18 @@ def handler(args: argparse.Namespace, _gbp: GBP, console: Console) -> int:
         console.err.print(f"{error.args[0]} not found.")
         return 1
 
+    def verbose_callback(phase: DumpPhase, build: Build) -> None:
+        console.err.print(f"dumping {phase} for {build}")
+
     filename = args.filename
     is_stdout = filename == "-"
+    kwargs = {"callback": verbose_callback} if args.verbose else {}
 
     try:
         # I'm using try/finally. Leave me alone pylint!
         # pylint: disable=consider-using-with
         fp = sys.stdout.buffer if is_stdout else open(filename, "wb")
-        publisher.dump(builds, fp)
+        publisher.dump(builds, fp, **kwargs)
     finally:
         if not is_stdout:
             fp.close()
@@ -41,6 +46,13 @@ def handler(args: argparse.Namespace, _gbp: GBP, console: Console) -> int:
 
 def parse_args(parser: argparse.ArgumentParser) -> None:
     """Set subcommand arguments"""
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        default=False,
+        help="verbose mode: list builds dumped",
+    )
     parser.add_argument(
         "filename", help='Filename to dump builds to ("-" for standard out)'
     )

src/gentoo_build_publisher/records/__init__.py

@@ -9,7 +9,12 @@
 from typing import IO, Any, Iterable, Protocol, Self
 
 from gentoo_build_publisher.settings import Settings
-from gentoo_build_publisher.types import ApiKey, Build
+from gentoo_build_publisher.types import (
+    ApiKey,
+    Build,
+    DumpCallback,
+    default_dump_callback,
+)
 from gentoo_build_publisher.utils import serializable
 
 
@@ -101,7 +106,13 @@ def count(self, machine: str | None = None) -> int:
         If `machine` is given, return the total number of builds for the given machine
         """
 
-    def dump(self, builds: Iterable[BuildRecord], outfile: IO[bytes]) -> None:
+    def dump(
+        self,
+        builds: Iterable[BuildRecord],
+        outfile: IO[bytes],
+        *,
+        callback: DumpCallback | None = None,
+    ) -> None:
         """Dump the given BuildRecords as JSON to the given file
 
         The JSON structure is an array of dataclasses.asdict(BuildRecord)
@@ -174,8 +185,16 @@ def from_settings(cls: type[Self], settings: Settings) -> Self:
         return cls(api_keys=api_keys(settings), build_records=build_records(settings))
 
 
-def dump_build_records(builds: Iterable[BuildRecord], outfile: IO[bytes]) -> None:
+def dump_build_records(
+    builds: Iterable[BuildRecord],
+    outfile: IO[bytes],
+    *,
+    callback: DumpCallback = default_dump_callback,
+) -> None:
     """Dump the given builds as JSON to the given file"""
+    for build in (builds := list(builds)):
+        callback("records", build)
+
     build_list = [asdict(build) for build in builds]
 
     serialized = json.dumps(build_list, default=serializable)

src/gentoo_build_publisher/records/django_orm.py

@@ -15,7 +15,12 @@
     RecordNotFound,
     dump_build_records,
 )
-from gentoo_build_publisher.types import ApiKey, Build
+from gentoo_build_publisher.types import (
+    ApiKey,
+    Build,
+    DumpCallback,
+    default_dump_callback,
+)
 from gentoo_build_publisher.utils import decode, decrypt, encode, encrypt
 
 RELATED = ("buildlog", "buildnote", "keptbuild")
@@ -207,12 +212,17 @@ def count(machine: str | None = None) -> int:
         return _manager.filter(**field_lookups).count()
 
     @staticmethod
-    def dump(builds: Iterable[BuildRecord], outfile: IO[bytes]) -> None:
+    def dump(
+        builds: Iterable[BuildRecord],
+        outfile: IO[bytes],
+        *,
+        callback: DumpCallback = default_dump_callback,
+    ) -> None:
         """Dump the given BuildRecords as JSON to the given file
 
         The JSON structure is an array of dataclasses.asdict(BuildRecord)
         """
-        dump_build_records(builds, outfile)
+        dump_build_records(builds, outfile, callback=callback)
 
 
 class ApiKeyDB:

src/gentoo_build_publisher/records/memory.py

@@ -9,7 +9,12 @@
     RecordNotFound,
     dump_build_records,
 )
-from gentoo_build_publisher.types import ApiKey, Build
+from gentoo_build_publisher.types import (
+    ApiKey,
+    Build,
+    DumpCallback,
+    default_dump_callback,
+)
 
 BuildId = str
 Machine = str
@@ -181,12 +186,17 @@ def count(self, machine: str | None = None) -> int:
         return sum(len(builds) for builds in self.builds.values())
 
     @staticmethod
-    def dump(builds: t.Iterable[BuildRecord], outfile: t.IO[bytes]) -> None:
+    def dump(
+        builds: t.Iterable[BuildRecord],
+        outfile: t.IO[bytes],
+        *,
+        callback: DumpCallback = default_dump_callback,
+    ) -> None:
         """Dump the given BuildRecords as JSON to the given file
 
         The JSON structure is an array of dataclasses.asdict(BuildRecord)
         """
-        dump_build_records(builds, outfile)
+        dump_build_records(builds, outfile, callback=callback)
 
 
 def record_key(record: BuildRecord) -> int | str:

src/gentoo_build_publisher/storage.py

@@ -18,9 +18,11 @@
     TAG_SYM,
     Build,
     Content,
+    DumpCallback,
     GBPMetadata,
     Package,
     PackageMetadata,
+    default_dump_callback,
 )
 
 INVALID_TEST_PATH = "__testing__"
@@ -284,14 +286,21 @@ def get_packages(self, build: Build) -> list[Package]:
 
             return list(make_packages(package_index_file))
 
-    def dump(self, builds: Iterable[Build], fp: IO[bytes]) -> None:
+    def dump(
+        self,
+        builds: Iterable[Build],
+        fp: IO[bytes],
+        *,
+        callback: DumpCallback = default_dump_callback,
+    ) -> None:
         """Dump the given builds' contents to the given file object
 
         The bytes dumped will be a tar archive. This includes any tags associated with
         the build.
         """
         with tar.open(fileobj=fp, mode="w") as tarfile, fs.cd(self.root):
             for build in builds:
+                callback("storage", build)
                 for content in Content:
                     for tag in [None, *self.get_tags(build)]:
                         path = self.get_path(build, content, tag=tag)

src/gentoo_build_publisher/types.py

@@ -5,7 +5,7 @@
 import datetime as dt
 from dataclasses import dataclass, field
 from enum import Enum, unique
-from typing import Any, Protocol
+from typing import Any, Callable, Literal, Protocol, TypeAlias
 
 from gentoo_build_publisher import utils
 
@@ -170,3 +170,11 @@ class ApiKey:
 
     def __post_init__(self) -> None:
         utils.validate_identifier(self.name)
+
+
+DumpPhase: TypeAlias = Literal["storage"] | Literal["records"]
+DumpCallback: TypeAlias = Callable[[DumpPhase, Build], Any]
+
+
+def default_dump_callback(_phase: DumpPhase, _build: Build) -> None:
+    """Default DumpCallback. A noop"""

tests/test_cli_dump.py

@@ -95,6 +95,30 @@ def test_dump_to_stdout(self) -> None:
 
         self.assertEqual(6, len(records(path)))
 
+    def test_verbose_flag(self) -> None:
+        builds = create_builds()
+        builds.sort(key=lambda build: (build.machine, build.build_id))
+
+        cmdline = "gbp dump -v -"
+
+        args = parse_args(cmdline)
+        gbp = mock.Mock()
+        console = self.fixtures.console
+
+        with mock.patch("gentoo_build_publisher.cli.dump.sys.stdout") as stdout:
+            stdout.buffer = io.BytesIO()
+            status = dump(args, gbp, console)
+
+        self.assertEqual(0, status)
+        expected = (
+            "\n".join(f"dumping storage for {build}" for build in builds)
+            + "\n"
+            + "\n".join(f"dumping records for {build}" for build in builds)
+            + "\n"
+        )
+
+        self.assertEqual(expected, console.err.file.getvalue())
+
     def test_build_id_not_found(self) -> None:
         create_builds()