refactor: move archive into a seperate subpackage

Instead of BuildPublisher, Storage, and Records gaining dump/restore
functionality, add a seperate subpackage, archive, that uses
BuildPublisher, Storage, and Records to do its task.

I'm thinking to eventually spin this off as a seperate plugin.

src/gentoo_build_publisher/build_publisher.py

@@ -20,11 +20,9 @@
 
 import logging
 import math
-import tarfile as tar
-import tempfile
 from datetime import datetime
 from difflib import Differ
-from typing import IO, Any, Iterable
+from typing import Any, Iterable
 
 from gentoo_build_publisher.jenkins import Jenkins, JenkinsMetadata
 from gentoo_build_publisher.machines import MachineInfo
@@ -37,21 +35,18 @@
     Build,
     Change,
     ChangeState,
-    DumpCallback,
     GBPMetadata,
     Package,
     PackageMetadata,
-    default_dump_callback,
 )
 from gentoo_build_publisher.utils.time import utctime
 
 logger = logging.getLogger(__name__)
 
 
-class BuildPublisher:  # pylint: disable=too-many-public-methods
+class BuildPublisher:
     """Pulls a build's db, jenkins and storage all together"""
 
-    # pylint: disable=redefined-outer-name
     def __init__(self, *, jenkins: Jenkins, storage: Storage, repo: Repo):
         self.jenkins = jenkins
         self.storage = storage
@@ -247,50 +242,6 @@ def latest_build(self, machine: str, completed: bool = False) -> BuildRecord | N
         """Return the latest completed build for the given machine name"""
         return self.repo.build_records.latest(machine, completed)
 
-    def dump(
-        self,
-        builds: Iterable[Build],
-        outfile: IO[bytes],
-        *,
-        callback: DumpCallback = default_dump_callback,
-    ) -> None:
-        """Dump the given builds to the given outfile"""
-        builds = list(builds)
-        builds.sort(key=lambda build: (build.machine, build.build_id))
-
-        with tar.open(fileobj=outfile, mode="w|") as tarfile:
-            # first dump records
-            with tempfile.SpooledTemporaryFile(mode="w+b") as tmp:
-                records = [self.repo.build_records.get(build) for build in builds]
-                self.repo.build_records.dump(records, tmp, callback=callback)
-                tmp.seek(0)
-                tarinfo = tarfile.gettarinfo(arcname="records.json", fileobj=tmp)
-                tarfile.addfile(tarinfo, tmp)
-
-            # then dump storage
-            with tempfile.TemporaryFile(mode="w+b") as tmp:
-                self.storage.dump(builds, tmp, callback=callback)
-                tmp.seek(0)
-                tarinfo = tarfile.gettarinfo(arcname="storage.tar", fileobj=tmp)
-                tarfile.addfile(tarinfo, tmp)
-
-    def restore(
-        self, infile: IO[bytes], *, callback: DumpCallback = default_dump_callback
-    ) -> None:
-        """Restore builds from the given infile"""
-        with tar.open(fileobj=infile, mode="r|") as tarfile:
-            for member in tarfile:
-                if member.name == "records.json":
-                    records_dump = tarfile.extractfile(member)
-                    assert records_dump is not None
-                    self.repo.build_records.restore(records_dump, callback=callback)
-                    continue
-                if member.name == "storage.tar":
-                    storage_dump = tarfile.extractfile(member)
-                    assert storage_dump is not None
-                    self.storage.restore(storage_dump, callback=callback)
-                    continue
-
     @staticmethod
     def gbp_metadata(
         jenkins_metadata: JenkinsMetadata, packages: list[Package]

src/gentoo_build_publisher/cli/dump.py

@@ -9,6 +9,7 @@
 from gentoo_build_publisher import publisher
 from gentoo_build_publisher.records import BuildRecord
 from gentoo_build_publisher.types import Build, DumpPhase, DumpType
+from gentoo_build_publisher.utils import archive
 
 HELP = "Dump builds to a file"
 
@@ -36,7 +37,7 @@ def verbose_callback(_type: DumpType, phase: DumpPhase, build: Build) -> None:
         # I'm using try/finally. Leave me alone pylint!
         # pylint: disable=consider-using-with
         fp = sys.stdout.buffer if is_stdout else open(filename, "wb")
-        publisher.dump(builds, fp, **kwargs)
+        archive.dump(builds, fp, **kwargs)
     finally:
         if not is_stdout:
             fp.close()

src/gentoo_build_publisher/cli/restore.py

@@ -6,8 +6,8 @@
 from gbpcli.gbp import GBP
 from gbpcli.types import Console
 
-from gentoo_build_publisher import publisher
 from gentoo_build_publisher.types import Build, DumpPhase, DumpType
+from gentoo_build_publisher.utils import archive
 
 HELP = "Restore a gbp dump"
 
@@ -26,7 +26,7 @@ def verbose_callback(_type: DumpType, phase: DumpPhase, build: Build) -> None:
         # I'm using try/finally. Leave me alone pylint!
         # pylint: disable=consider-using-with
         fp = sys.stdin.buffer if is_stdin else open(filename, "rb")
-        publisher.restore(fp, **kwargs)
+        archive.restore(fp, **kwargs)
     finally:
         if not is_stdin:
             fp.close()

src/gentoo_build_publisher/records/__init__.py

@@ -1,21 +1,12 @@
 """DB interface for Gentoo Build Publisher"""
 
-from __future__ import annotations
-
 import datetime as dt
 import importlib.metadata
-import json
-from dataclasses import asdict, dataclass
-from typing import IO, Any, Iterable, Protocol, Self
+from dataclasses import dataclass
+from typing import Any, Iterable, Protocol, Self
 
 from gentoo_build_publisher.settings import Settings
-from gentoo_build_publisher.types import (
-    ApiKey,
-    Build,
-    DumpCallback,
-    default_dump_callback,
-)
-from gentoo_build_publisher.utils import convert_to, decode_to, serializable
+from gentoo_build_publisher.types import ApiKey, Build
 
 
 class RecordNotFound(LookupError):
@@ -106,30 +97,6 @@ def count(self, machine: str | None = None) -> int:
         If `machine` is given, return the total number of builds for the given machine
         """
 
-    def dump(
-        self,
-        builds: Iterable[BuildRecord],
-        outfile: IO[bytes],
-        *,
-        callback: DumpCallback = default_dump_callback,
-    ) -> None:
-        """Dump the given BuildRecords as JSON to the given file
-
-        The JSON structure is an array of dataclasses.asdict(BuildRecord)
-
-        See also dump_build_records below which is a function that already does this.
-        """
-
-    def restore(
-        self, infile: IO[bytes], *, callback: DumpCallback = default_dump_callback
-    ) -> list[BuildRecord]:
-        """Restore to the db the records given in the infile
-
-        The infile should be structured with the dump() method.
-
-        See also restore_build_records below which is a function that already does this.
-        """
-
 
 def build_records(settings: Settings) -> RecordDB:
     """Return instance of the the RecordDB class given in settings"""
@@ -193,48 +160,3 @@ class Repo:
     def from_settings(cls: type[Self], settings: Settings) -> Self:
         """Return instance of the the Repo class given in settings"""
         return cls(api_keys=api_keys(settings), build_records=build_records(settings))
-
-
-def dump_build_records(
-    builds: Iterable[BuildRecord],
-    outfile: IO[bytes],
-    *,
-    callback: DumpCallback = default_dump_callback,
-) -> None:
-    """Dump the given builds as JSON to the given file"""
-    for build in (builds := list(builds)):
-        callback("dump", "records", build)
-
-    build_list = [asdict(build) for build in builds]
-
-    serialized = json.dumps(build_list, default=serializable)
-    outfile.write(serialized.encode("utf8"))
-
-
-@convert_to(BuildRecord, "built")
-@convert_to(BuildRecord, "completed")
-@convert_to(BuildRecord, "submitted")
-def _(value: str | None) -> dt.datetime | None:
-    return None if value is None else dt.datetime.fromisoformat(value)
-
-
-def restore_build_records(
-    infile: IO[bytes],
-    records: RecordDB,
-    *,
-    callback: DumpCallback = default_dump_callback,
-) -> list[BuildRecord]:
-    """Restore the JSON given in the infile to BuildRecords in the given RecordDB
-
-    Return the restored records
-    """
-    restore_list: list[BuildRecord] = []
-
-    items = json.load(infile)
-    for item in items:
-        record = decode_to(BuildRecord, item)
-        callback("restore", "records", record)
-        record = records.save(record)
-        restore_list.append(record)
-
-    return restore_list

src/gentoo_build_publisher/records/django_orm.py

@@ -2,26 +2,16 @@
 
 import datetime as dt
 from dataclasses import replace
-from typing import IO, Any, Iterable
+from typing import Any, Iterable
 
 from django.conf import settings
 from django.db import models
 
 import gentoo_build_publisher._django_setup  # pylint: disable=unused-import
 from gentoo_build_publisher.models import ApiKey as ApiKeyModel
 from gentoo_build_publisher.models import BuildLog, BuildModel, BuildNote, KeptBuild
-from gentoo_build_publisher.records import (
-    BuildRecord,
-    RecordNotFound,
-    dump_build_records,
-    restore_build_records,
-)
-from gentoo_build_publisher.types import (
-    ApiKey,
-    Build,
-    DumpCallback,
-    default_dump_callback,
-)
+from gentoo_build_publisher.records import BuildRecord, RecordNotFound
+from gentoo_build_publisher.types import ApiKey, Build
 from gentoo_build_publisher.utils import decode, decrypt, encode, encrypt
 
 RELATED = ("buildlog", "buildnote", "keptbuild")
@@ -212,28 +202,6 @@ def count(machine: str | None = None) -> int:
 
         return _manager.filter(**field_lookups).count()
 
-    @staticmethod
-    def dump(
-        builds: Iterable[BuildRecord],
-        outfile: IO[bytes],
-        *,
-        callback: DumpCallback = default_dump_callback,
-    ) -> None:
-        """Dump the given BuildRecords as JSON to the given file
-
-        The JSON structure is an array of dataclasses.asdict(BuildRecord)
-        """
-        dump_build_records(builds, outfile, callback=callback)
-
-    def restore(
-        self, infile: IO[bytes], *, callback: DumpCallback = default_dump_callback
-    ) -> list[BuildRecord]:
-        """Restore to the db the records given in the infile
-
-        The infile should be structured with the dump() method.
-        """
-        return restore_build_records(infile, self, callback=callback)
-
 
 class ApiKeyDB:
     """Implements the ApiKeyDB Protocol using Django's ORM as a backing store"""

src/gentoo_build_publisher/records/memory.py

@@ -4,18 +4,8 @@
 import typing as t
 from dataclasses import replace
 
-from gentoo_build_publisher.records import (
-    BuildRecord,
-    RecordNotFound,
-    dump_build_records,
-    restore_build_records,
-)
-from gentoo_build_publisher.types import (
-    ApiKey,
-    Build,
-    DumpCallback,
-    default_dump_callback,
-)
+from gentoo_build_publisher.records import BuildRecord, RecordNotFound
+from gentoo_build_publisher.types import ApiKey, Build
 
 BuildId = str
 Machine = str
@@ -186,28 +176,6 @@ def count(self, machine: str | None = None) -> int:
 
         return sum(len(builds) for builds in self.builds.values())
 
-    def dump(
-        self,
-        builds: t.Iterable[BuildRecord],
-        outfile: t.IO[bytes],
-        *,
-        callback: DumpCallback = default_dump_callback,
-    ) -> None:
-        """Dump the given BuildRecords as JSON to the given file
-
-        The JSON structure is an array of dataclasses.asdict(BuildRecord)
-        """
-        dump_build_records(builds, outfile, callback=callback)
-
-    def restore(
-        self, infile: t.IO[bytes], *, callback: DumpCallback = default_dump_callback
-    ) -> list[BuildRecord]:
-        """Restore to the db the records given in the infile
-
-        The infile should be structured with the dump() method.
-        """
-        return restore_build_records(infile, self, callback=callback)
-
 
 def record_key(record: BuildRecord) -> int | str:
     """Sort key function for records (of the same machine)"""