pkgbuild

#!/usr/bin/env python3

import argparse
import atexit
from concurrent.futures import ThreadPoolExecutor, wait, FIRST_COMPLETED
from configparser import ConfigParser
from datetime import datetime
from email.utils import parsedate_to_datetime
import filetype
from hashlib import sha256
import json
from os import cpu_count, getegid, geteuid, link
from pathlib import Path
import re
import socket
from shlex import quote
from shutil import copy2, copytree, rmtree
from subprocess import Popen, CalledProcessError, PIPE, STDOUT
from sys import argv, stderr
from string import Template
from tempfile import mkdtemp, TemporaryDirectory
from time import sleep, time
from typing import Iterable, Set
from urllib.error import URLError
from urllib.parse import parse_qsl, ParseResult, urljoin, urlparse, urlunparse
from urllib.request import url2pathname, urlopen

from debian.changelog import Changelog, format_date
from debian.deb822 import Deb822, _PkgRelationMixin
import yaml

from downloader import Downloader
from distinfo import ReleaseInfo, distributions

VERBOSE = None
REPO = 'ghcr.io/jbigot/pkg_builder/'


def urlnormalize(url):
    su = urlparse(url)
    su = ParseResult(su.scheme, su.netloc, str(Path(su.path)),
                     su.params, su.query, su.fragment)
    return urlunparse(su)


class GpgInfo:
    def __init__(self, keyfile, keyid=None, passphrase=None, uid=None):
        self._secring = keyfile
        self._keyid = keyid
        self._passphrase = passphrase
        self._home_dir = Path(mkdtemp(prefix='GNUPG_HOME.'))
        atexit.register(rmtree, self._home_dir)
        self._gpg_bin = self._home_dir / 'bin' / 'gpg'
        self._uid = uid

        if self._keyid is not None:
            self._keyid = self._keyid.upper()

        self._home_dir.chmod(0o700)
        cheched_run(['gpg2', '--batch', '--homedir', self.home()] +
                    self.passphrase('', ['--passphrase']) +
                    ['--import', self.ring()],
                    cwd=self._home_dir)

        self._gpg_bin.parent.mkdir()
        with self._gpg_bin.open('w') as gpg_bin_raw:
            print('#!/bin/sh', file=gpg_bin_raw)
            print('exec gpg2 --batch --pinentry-mode loopback --homedir '
                  + quote(self.home()) + ' --passphrase '
                  + quote(self.passphrase('')) + ' "$@"', file=gpg_bin_raw)
        self._gpg_bin.chmod(0o700)

        data = cheched_run(['gpg2', '--batch', '--homedir', self.home(),
                            '--with-colons', '--with-fingerprint',
                            '--fixed-list-mode'] + self.passphrase(
                               '', ['--passphrase']) + ['--list-secret-keys'],
                           cwd=self._home_dir, return_stdout=True)
        in_key = False
        for line in data.splitlines():
            res = line.split(':')
            if res[0] == 'sec':
                in_key = False
                if self._keyid is None or res[4].upper(
                )[-8:] == self._keyid[-8:]:
                    in_key = True
            elif (res[0] == 'ssb' or res[0] == 'pub' or res[0] == 'crt'
                  or res[0] == 'crs' or res[0] == 'sub'):
                in_key = False
            elif res[0] == 'fpr' and in_key:
                self._keyid = res[9].upper()
            elif res[0] == 'uid' and in_key:
                if self._uid is None or res[9].strip() == self._uid.strip():
                    self._uid = res[9]

    def _combine(self, prel, pre, data, post, postl):
        if data is None:
            if prel is not None or postl is not None:
                return []
            if pre is not None or post is not None:
                return ''
            return None
        data = str(data)
        if pre is not None:
            data = pre + data
        if post is not None:
            data = data + post
        if prel is not None or postl is not None:
            data = [data]
        if prel is not None:
            data = prel + data
        if postl is not None:
            data = data + postl
        return data

    def uid(self, pre=None, prel=None, post=None, postl=None):
        return self._combine(prel, pre, self._uid, post, postl)

    def id(self, pre=None, prel=None, post=None, postl=None):
        return self._combine(prel, pre, self._keyid, post, postl)

    def id8(self, pre=None, prel=None, post=None, postl=None):
        return self._combine(prel, pre, self._keyid[-8:], post, postl)

    def passphrase(self, pre=None, prel=None, post=None, postl=None):
        return self._combine(prel, pre, self._passphrase, post, postl)

    def ring(self, pre=None, prel=None, post=None, postl=None):
        return self._combine(prel, pre, self._secring, post, postl)

    def bin(self, pre=None, prel=None, post=None, postl=None):
        return self._combine(prel, pre, self._gpg_bin, post, postl)

    def home(self, pre=None, prel=None, post=None, postl=None):
        return self._combine(prel, pre, self._home_dir, post, postl)


RUNNING_PROCESSES = set()
TERMINATE_REQUEST = False


class CancellationException(Exception):
    pass


def cancellation_point():
    if TERMINATE_REQUEST:
        raise CancellationException('Termination request received')


def cancellation_request():
    global TERMINATE_REQUEST
    TERMINATE_REQUEST = True
    for process in RUNNING_PROCESSES:
        if process.poll() is None:
            process.terminate()


def cheched_run(cmd, cwd=None, return_stdout=False):
    with TemporaryDirectory() as tmp:
        if cwd is None:
            cwd = tmp
        err = STDOUT
        if VERBOSE:
            out = None
            print("$ " + ' '.join([str(c) for c in cmd]))
        else:
            out = PIPE
        if return_stdout:
            err = out
            out = PIPE
        cancellation_point()
        with Popen(cmd, cwd=cwd, stdout=out, stderr=err,
                   encoding='UTF8') as process:
            try:
                RUNNING_PROCESSES.add(process)
                out, err = process.communicate()
            except BaseException:
                process.kill()
                raise
            finally:
                RUNNING_PROCESSES.remove(process)
                cancellation_point()
            retcode = process.poll()
            if retcode:
                raise CalledProcessError(
                    retcode, process.args, output=out, stderr=err)
            if return_stdout:
                if VERBOSE:
                    print(out, end='')
                return out
    return None


class Control(Deb822, _PkgRelationMixin):
    _relationship_fields = ['build-depends',
                            'build-depends-indep', 'build-depends-arch']

    def __init__(self, *args, **kwargs):
        Deb822.__init__(self, *args, **kwargs)
        _PkgRelationMixin.__init__(self, *args, **kwargs)


class DebPackageBuilder:
    def __init__(self, dist_release: ReleaseInfo, base_dir: Path, gpg: GpgInfo, name: str,
                 downloader: Downloader, orig_url: str, parallelism: int):
        self.dist_release = dist_release
        self._debian_dir = base_dir / name / 'debian'
        self._gpg = gpg
        self._name = name
        self._downloader = downloader
        self._orig_url = orig_url
        self._parallelism = parallelism

        self._finished = False
        self.outdir = None
        self._depends = set([self])  # self depend => never ready
        self.bin_provides = set()
        self.bin_depends = set()

        if not self._debian_dir.is_dir():
            self._debian_dir = None
            return

        deps_kind = ['build-depends',
                     'build-depends-indep', 'build-depends-arch']
        with (self._debian_dir / 'control').open() as raw_control:
            control_paragraphs = Deb822.iter_paragraphs(
                raw_control, strict={'whitespace-separates-paragraphs': False})
            self.bin_provides = {pkg['Package'] for
                                 pkg in control_paragraphs if 'Package' in pkg}
        with (self._debian_dir / 'control').open() as raw_control:
            control = Control(raw_control)
            self.bin_depends = {om['name'] for
                                dl in deps_kind for am in control.relations[dl] for om in am}

    def ready(self):
        return all(map(DebPackageBuilder.finished, self._depends))

    def finished(self):
        return self._finished

    def __dep_packages(self):
        return {e for el in self._depends for e in el.__dep_packages()} | set([
            self])

    def build(self, wk_dir):
        cancellation_point()
        try:
            print("\033[34m[STARTED]\033[39m * building deb for " +
                  self._name + ' on ' + str(self.dist_release) + "...")

            if self._debian_dir is None:
                self._finished = True
                print("\033[36m[SKIPPED]\033[39m * building deb for " +
                      self._name + ' on ' + str(self.dist_release) + "!")
                return

            # Setup our directories
            wk_dir = wk_dir / (self._name + '.' +
                               self.dist_release.uid() + '.deb-build')
            wk_dir.mkdir()
            self.outdir = wk_dir / 'output'
            self.outdir.mkdir()
            pkg_dir = wk_dir / 'pkg'
            pkg_dir.mkdir()

            # Copy the data
            fmt = cheched_run(['dpkg-source', '--print-format', str(self._debian_dir.parent)],
                              return_stdout=True).strip()
            with (self._debian_dir / 'changelog').open() as raw_changelog:
                changelog = Changelog(raw_changelog)
            if fmt == '3.0 (quilt)':
                pkg_src_dir = pkg_dir / \
                    (changelog.package + '-' + changelog.upstream_version)
                debian_dir = Path(
                    copytree(self._debian_dir, pkg_src_dir / 'debian'))
            elif fmt == '3.0 (native)':
                pkg_src_dir = Path(copytree(self._debian_dir.parent,
                                            pkg_dir / (changelog.package + '-'
                                                       + changelog.upstream_version)))
                debian_dir = pkg_src_dir / 'debian'
            else:
                raise Exception('Unsupported deb package format: ' + fmt)

            # Generate the automated entry in changelog
            version = str(changelog.version)
            if self.dist_release.id is not None and self.dist_release.id != '':
                version += '~bpo' + self.dist_release.id
            chlg_date = parsedate_to_datetime(changelog.date)
            chlg_date = (datetime.now(tz=chlg_date.tzinfo) - chlg_date)
            version += '.pdidev.' + str(int(chlg_date.total_seconds()))
            changelog.new_block(
                package=changelog.package,
                version=version,
                distributions=self.dist_release.codename,
                urgency=changelog.urgency,
                changes=['  * Rebuild for ' + str(self.dist_release)],
                author=self._gpg.uid(),
                date=format_date())
            with (debian_dir / 'changelog').open('w') as raw_changelog:
                changelog.write_to_open_file(raw_changelog)

            # Download the orig file
            if fmt == '3.0 (quilt)':
                self._orig_url = self._orig_url.format(
                    upstream_version=changelog.upstream_version, package=changelog.package)
                orig_file_noext = (changelog.package + '_' +
                                   changelog.upstream_version + '.orig.tar.')
                self._downloader.download(
                    self._orig_url, pkg_dir / orig_file_noext)
                orig_file = orig_file_noext + \
                    filetype.guess(str(pkg_dir / orig_file_noext)).extension
                (pkg_dir / orig_file_noext).rename(pkg_dir / orig_file)

            # Generate the dependencies .deb
            deps_dir = wk_dir / 'deps'
            deps_dir.mkdir()
            cheched_run(['mk-build-deps', '-P' + self.dist_release.distrib.id + ',' +
                         self.dist_release.codename, debian_dir / 'control'], cwd=deps_dir)

            # Build a repo with all dependencies
            localrepo = wk_dir / 'repo'
            DebRepoBuilder(localrepo, self._gpg).build(self.__dep_packages())

            # Build the package in docker
            cheched_run(['docker', 'run', '--rm', '-eDH_VERBOSE=1', '--shm-size=5g',
                         '--mount', 'type=bind,source=' +
                         str(pkg_dir) + ',target=/src',
                         '--mount', 'type=bind,source=' +
                         str(deps_dir) + ',target=/deps',
                         '--mount', 'type=bind,source=' +
                         str(localrepo) + ',target=/localrepo',
                         '--tmpfs', '/tmp:exec',
                         REPO + '' + self.dist_release.distrib.id +
                         '_builder:' + self.dist_release.codename,
                         '-j' + str(self._parallelism), '-sa',
                         '-P' + self.dist_release.distrib.id +
                         ',' + self.dist_release.codename
                         ])
            rmtree(localrepo)
            rmtree(deps_dir)

            # Signing the generated packages
            cheched_run(['debsign', '--no-conf', self._gpg.bin('-p')] + self._gpg.id('-k', []) + [
                '--debs-dir', str(pkg_dir)], cwd=debian_dir.parent)

            # Moving the generated packages to the output directory
            for file in pkg_dir.iterdir():
                if file.is_file():
                    file.rename(self.outdir / file.name)
            rmtree(pkg_dir)

            self._finished = True
            print("\033[32m[SUCCESS]\033[39m * building deb for " +
                  self._name + ' on ' + str(self.dist_release) + "!")
        except CancellationException:
            print("\033[33m[CANCEL]\033[39m  * building deb for " +
                  self._name + ' on ' + str(self.dist_release) + "!")
            raise
        except BaseException as e:
            print("\033[31m[FAILURE]\033[39m * building deb for " +
                  self._name + ' on ' + str(self.dist_release) + "! " + str(e))
            raise


class DebRepoBuilder:
    def __init__(self, output_dir: Path, gpg: GpgInfo, url: str = None, name: str = None,
                 description: str = None, readme_tpl: Path = None):
        self._output_dir = output_dir
        self._gpg = gpg
        self._url = url
        if url is not None:
            self._url = urlnormalize(self._url)
            self._name = name if name is not None else Path(
                str(urlparse(self._url).path)).name
            self._description = description if description is not None else name
        self._readme_tpl = readme_tpl

    def build(self, pkgs: Iterable[DebPackageBuilder]):
        pkgs = list(
            filter(lambda p: 'debian' in p.dist_release.distrib.id_like, pkgs))

        if len(pkgs) == 0:
            return

        distribs = set()
        install = ""
        with TemporaryDirectory() as tmp_dir:
            tmp_dir = Path(tmp_dir)
            aptly_confpath = tmp_dir / 'aptly.conf'

            def aptly(*args):
                cheched_run(
                    ['aptly', '-config=' + str(aptly_confpath)] + list(args))

            aptly_cfg = {
                'architectures': ["amd64", "source"],
                'gpgProvider': "internal",
                'rootDir': str(tmp_dir),
                'FileSystemPublishEndpoints': {'default': {
                    'rootDir': str(self._output_dir),
                    'linkMethod': "copy",
                }},
            }
            with aptly_confpath.open('w') as aptly_conf:
                json.dump(aptly_cfg, aptly_conf)

            self._output_dir.mkdir(parents=True, exist_ok=True)

            repos = set()
            for pkg in pkgs:
                if pkg.dist_release not in repos:
                    repos.add(pkg.dist_release)
                    aptly('repo', 'create', '-distribution=' +
                          pkg.dist_release.codename, pkg.dist_release.uid())
                    if pkg.dist_release.suite is not None:
                        aptly('repo', 'create', '-distribution=' + pkg.dist_release.suite,
                              pkg.dist_release.uid() + ':' + pkg.dist_release.suite)
                aptly('repo', 'include', self._gpg.ring('-keyring='),
                      '-no-remove-files', '-repo=' + pkg.dist_release.uid(), str(pkg.outdir))
                if pkg.dist_release.suite is not None:
                    aptly('repo', 'include', self._gpg.ring('-keyring='), '-no-remove-files',
                          '-repo=' + pkg.dist_release.uid() + ':' + pkg.dist_release.suite,
                          str(pkg.outdir))
            for dist_release in repos:
                distribs.add(str(dist_release.distrib))
                if self._readme_tpl is not None:
                    with open(self._readme_tpl / ('INSTALL.' + str(dist_release.distrib.id)
                                                  + '.tpl.html')) as template_file:
                        install += Template(
                            template_file.read()).substitute(
                            dist_release=str(dist_release), codename=str(
                                dist_release.codename), baseurl=str(
                                self._url))
                publish_repo = (['publish', 'repo', '-batch', self._gpg.ring('-secret-keyring='),
                                 '-force-overwrite'] + self._gpg.id8('-gpg-key=', []) +
                                self._gpg.passphrase('-passphrase=', []))
                if self._url is not None:
                    ['-notautomatic=yes', '-butautomaticupgrades=yes']
                    publish_repo += ['-label=' +
                                     self._description, '-origin=' + self._name]

                aptly(*publish_repo, dist_release.uid(),
                      'filesystem:default:')
                if dist_release.suite is not None:
                    aptly(*publish_repo, dist_release.uid() + ':' + dist_release.suite,
                          'filesystem:default:')

        if self._url is not None:
            (self._output_dir / (self._name + '-archive-keyring.gpg')
             ).unlink(missing_ok=True)
            cheched_run(
                [self._gpg.bin(),
                 '--export', '--output',
                 str(self._output_dir / (self._name + '-archive-keyring.gpg'))],
                cwd=self._output_dir)
        if self._readme_tpl is not None:
            with open(self._output_dir / 'index.html', mode='w') as readme_file:
                with open(self._readme_tpl / 'README.tpl.html') as template_file:
                    readme_file.write(Template(template_file.read()).substitute(
                        distribs=" ".join(list(distribs)), install=install))


class RpmPackageBuilder:
    def __init__(self, dist_release: ReleaseInfo, base_dir: Path, gpg: GpgInfo, name: str,
                 downloader: Downloader, orig_url: str, parallelism: int):
        self.dist_release = dist_release
        self._spec = base_dir / name / (name + '.spec')
        self._gpg = gpg
        self._name = name
        self._downloader = downloader
        self._parallelism = parallelism

        self._finished = False
        self._depends = set([self])  # self depend => never ready
        self.outdir = None
        self.bin_provides = set()
        self.bin_depends = set()

        if not self._spec.is_file():
            self._spec = None
            return

        self.bin_depends = {re.sub(r'[\( ].*', '', dep) for dep in
                            cheched_run(['rpmspec', '--buildrequires', '-q', str(self._spec)],
                                        return_stdout=True).splitlines()}
        self.bin_provides = {re.sub(r'[\( ].*', '', d) for d in
                             cheched_run(['rpmspec', '--provides', '-q', str(self._spec)],
                                         return_stdout=True).splitlines()}

    def ready(self):
        return all(map(RpmPackageBuilder.finished, self._depends))

    def finished(self):
        return self._finished

    def incomings(self):
        return {e for el in self._depends for e in el.incomings()} | set([
            self])

    def build(self, wk_dir):
        cancellation_point()
        try:
            print("\033[34m[STARTED]\033[39m * building rpm for " +
                  self._name + ' on ' + str(self.dist_release) + "...")

            if self._spec is None:
                self._finished = True
                print("\033[36m[SKIPPED]\033[39m * building rpm for " +
                      self._name + ' on ' + str(self.dist_release) + "!")
                return

            # Setup our directories
            wk_dir = wk_dir / (self._name + '.' +
                               self.dist_release.uid() + '.rpm-build')
            wk_dir.mkdir()
            self.outdir = wk_dir / 'output'
            self.outdir.mkdir()
            pkg_dir = wk_dir / 'pkg'
            pkg_dir.mkdir()
            spec = pkg_dir / self._spec.name
            copy2(self._spec, spec)

            localrepo = wk_dir / 'repo'
            RpmRepoBuilder(localrepo, self._gpg).build(self.incomings())

            all_urls = cheched_run(['rpmspec', '-P', str(self._spec)],
                                   return_stdout=True).splitlines()
            all_urls = [url for url in all_urls if re.match(
                r'^\s*source[0-9]*\s*:', url, re.IGNORECASE) is not None]
            all_urls = [re.sub(r'^\s*source[0-9]*\s*:\s*',
                               '', url, 1, re.IGNORECASE) for url in all_urls]
            for orig_url in all_urls:
                query = parse_qsl(urlparse(orig_url).query)
                if len(query) > 0:
                    orig_file = Path(query[-1][1]).name
                else:
                    orig_file = Path(urlparse(orig_url).path).name
                self._downloader.download(orig_url, pkg_dir / orig_file)

            cheched_run(['docker', 'run', '--rm', '--shm-size=5g', '--mount', 'type=bind,source=' +
                         str(pkg_dir) +
                         ',target=/src', '--mount', 'type=bind,source=' +
                         str(localrepo / self.dist_release.id) +
                         ',target=/localrepo', '--tmpfs', '/tmp:exec', REPO + '' +
                         self.dist_release.distrib.id +
                         '_builder:' +
                         self.dist_release.id])
            rmtree(localrepo)

            # Sign the generated packages
            for file in pkg_dir.iterdir():
                if file.suffix == '.rpm':
                    cheched_run(['rpmsign', self._gpg.bin('--define=%_gpg_bin '),
                                 self._gpg.bin('--define=%__gpg '),
                                 self._gpg.home('--define=%_gpg_home '),
                                 self._gpg.uid('--define=%_gpg_name '),
                                 '--resign', file], cwd=pkg_dir)

            for file in pkg_dir.iterdir():
                if file.suffix == '.rpm':
                    file.rename(self.outdir / file.name)
            rmtree(pkg_dir)

            print("\033[32m[SUCCESS]\033[39m * building rpm for " +
                  self._name + ' on ' + str(self.dist_release) + "!")
            self._finished = True
        except CancellationException:
            print("\033[33m[CANCEL]\033[39m  * building rpm for " +
                  self._name + ' on ' + str(self.dist_release) + "!")
            raise
        except BaseException:
            print("\033[31m[FAILURE]\033[39m * building rpm for " +
                  self._name + ' on ' + str(self.dist_release) + "!")
            raise


class RpmRepoBuilder:
    def __init__(self, output_dir: Path, gpg: GpgInfo, url: str = None, name: str = None,
                 description: str = None, readme_tpl: Path = None):
        self._output_dir = output_dir
        self._gpg = gpg
        self._url = url
        if url is not None:
            self._name = name if name is not None else str(
                Path(urlparse(str(self._url)).path).name)
            self._description = description if description is not None else name
        self._readme_tpl = readme_tpl

    def build(self, pkgs: Iterable[RpmPackageBuilder]):
        pkgs = list(
            filter(lambda p: 'fedora' in p.dist_release.distrib.id_like, pkgs))
        if len(pkgs) == 0:
            return

        releases_set = set()
        self._output_dir.mkdir(parents=True, exist_ok=True)
        for pkg in pkgs:
            if pkg.outdir is None or not pkg.outdir.is_dir():
                continue
            release_dir = self._output_dir / pkg.dist_release.id
            release_dir.mkdir(parents=True, exist_ok=True)
            for file in pkg.outdir.iterdir():
                if file.suffix == '.rpm' and len(file.suffixes) >= 2:
                    releases_set.add(pkg.dist_release)
                    rpmdir = release_dir / file.suffixes[-2][1:]
                    rpmdir.mkdir(exist_ok=True)
                    try:
                        link(file, rpmdir / file.name)
                    except OSError:
                        copy2(file, rpmdir / file.name)

        if self._url is not None:
            (self._output_dir / (self._name + '.key')).unlink(missing_ok=True)
            cheched_run([self._gpg.bin(), '--armor', '--export', '--output',
                         str(self._output_dir / (self._name + '.key'))], cwd=self._output_dir)

        install = ""
        distribs = set()
        for dist_release in releases_set:
            release_dir = self._output_dir / dist_release.id

            distribs.add(str(dist_release.distrib))

            cheched_run(
                ['docker', 'run', '--rm', '--shm-size=5g', '--user=' + str(geteuid()) + ':' +
                 str(getegid()),
                 '--mount', 'type=bind,source=' +
                 str(release_dir) + ',target=/data',
                 REPO + 'rpm_tools', '.'])
            (release_dir / 'repodata' / 'repomd.xml.asc').unlink(missing_ok=True)
            cheched_run([self._gpg.bin(), '-b', '-a',
                         'repodata/repomd.xml'], cwd=release_dir)
            if self._url is not None:
                baseurl = urljoin(self._url, dist_release.id)
                repo_data = ConfigParser()
                repo_data.add_section(self._name)
                repo_data.set(self._name, 'name',
                              self._description + ' (' + str(dist_release) + ')')
                repo_data.set(self._name, 'type', 'rpm-md')
                repo_data.set(self._name, 'baseurl', baseurl)
                repo_data.set(self._name, 'gpgcheck', '1')
                repo_data.set(self._name, 'repo_gpgcheck', '1')
                repo_data.set(self._name, 'gpgkey', urljoin(
                    self._url, self._name + '.key'))
                repo_data.set(self._name, 'enabled', '1')
                with (release_dir / (self._name + '.repo')).open('w') as repo_file:
                    repo_data.write(repo_file, space_around_delimiters=False)
                if self._readme_tpl is not None:
                    with open(self._readme_tpl / ('INSTALL.' + str(dist_release.distrib.id)
                                                  + '.tpl.html')) as template_file:
                        install += Template(
                            template_file.read()).substitute(
                            dist_release=str(dist_release), codename=str(
                                dist_release.codename), baseurl=str(baseurl))

        if self._readme_tpl is not None:
            with open(self._output_dir / 'README.html', mode='w') as readme_file:
                with open(self._readme_tpl / 'README.tpl.html') as template_file:
                    readme_file.write(Template(template_file.read()).substitute(
                        distribs=" ".join(list(distribs)), install=install))


def order_packages(pkglst):
    # maps (b, r) => s when the binary package b is generated by the source
    # package s on release r
    providers = {(bpkg, pkg.dist_release): pkg for pkg in pkglst for bpkg in pkg.bin_provides}
    for pkg in pkglst:
        pkg._depends = {providers[(b, pkg.dist_release)] for b in pkg.bin_depends if (
            b, pkg.dist_release) in providers}


def release_list(cfg: dict) -> Set[ReleaseInfo]:
    releases = set()
    for dist_id, release_id_lst in cfg.items():
        dist = next(distributions(id=dist_id))
        if not isinstance(release_id_lst, list):
            release_id_lst = [release_id_lst]
        prev = set()
        for release_id in release_id_lst:
            cur = set()
            if release_id == 'all':
                cur.update(dist.releases())
            elif release_id == '+':
                cur.update(dist.releases(after=max(prev)))
            if release_id == '-':
                cur.update(dist.releases(before=min(prev)))
            elif release_id == 'supported':
                cur.update(dist.releases(supported=True))
            else:
                cur.update(dist.releases(codename=release_id))
                cur.update(dist.releases(suite=release_id))
                cur.update(dist.releases(id=release_id))
            releases |= cur
            prev = cur
    return releases


def release_filter(releases: Iterable[ReleaseInfo],
                   dist_filter: Iterable[str]) -> Set[ReleaseInfo]:
    result = set()
    for one_filter in dist_filter:
        for release in releases:
            if (
                release.distrib.name == one_filter
                or one_filter == (str(release.distrib.name) + ':' + str(release.id))
                or one_filter == (str(release.distrib.name) + ':' + str(release.name))
                or one_filter == (str(release.distrib.name) + ':' + str(release.suite))
                or one_filter == (str(release.distrib.name) + ':' + str(release.codename))
                or str(release.distrib.id) == str(one_filter)
                or one_filter == (str(release.distrib.id) + ':' + str(release.id))
                or one_filter == (str(release.distrib.id) + ':' + str(release.name))
                or one_filter == (str(release.distrib.id) + ':' + str(release.suite))
                or one_filter == (str(release.distrib.id) + ':' + str(release.codename))
            ):
                result.add(release)
    return result


def build_packages(packages, parallelism: int = None):
    with ThreadPoolExecutor(max_workers=parallelism) as executor:
        try:
            tmp_dir = Path(mkdtemp(prefix='PKGBUILD.'))
            atexit.register(rmtree, tmp_dir)
            order_packages(packages)

            in_progress = set()
            waiting = set(packages)
            while (len(waiting) + len(in_progress)) > 0:
                for pkg_cfg in {
                        pkg_cfg for pkg_cfg in waiting if pkg_cfg.ready()}:
                    waiting.remove(pkg_cfg)
                    in_progress.add(executor.submit(pkg_cfg.build, tmp_dir))
                if len(in_progress) == 0:
                    raise Exception('Invalid scheduling: no task ready')
                done, in_progress = wait(
                    in_progress, return_when=FIRST_COMPLETED)
                for done_pkg in done:
                    done_pkg.result()
        except BaseException:
            executor.shutdown(False)
            cancellation_request()
            raise


def main():
    base_dir = Path.cwd().absolute()

    parser = argparse.ArgumentParser(description='Builds deb & RPM repositories.',
                                     allow_abbrev=False)
    parser.add_argument('-p', '--passphrase', action='store',
                        help='The passphrase for the GPG release key')
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Wether to run in verbose mode')
    parser.add_argument('-i', '--interactive', action='store_true',
                        help='Wether to run in interactive mode')
    parser.add_argument('-D', '--distributions', action='append', nargs='+',
                        help='The distributions to build')
    parser.add_argument('-j', '--jobs', action='store', nargs='?', type=int, default=None,
                        help='Number of jobs to run in parallel')
    parser.add_argument('build_conf', action='store', metavar='build.conf', nargs='?',
                        default='build.conf', help='Build configuration file')
    args = parser.parse_args()

    global VERBOSE
    VERBOSE = args.verbose
    if args.jobs is None and args.verbose:
        args.jobs = 1
    elif args.jobs is None:
        args.jobs = cpu_count() + 1

    try:
        with (base_dir / args.build_conf).open('r') as cfg_file:
            cfg = yaml.safe_load(cfg_file)

        downloader = Downloader(verbose=args.verbose,
                                cancellation_point=cancellation_point)

        packages = set()
        repo_builders = []
        for dist_id, distrib_cfg in cfg['distribs'].items():
            keyfile = base_dir / \
                distrib_cfg.get('gpg', {}).get('file', 'key.gpg')
            keyid = distrib_cfg.get('gpg', {}).get('id', None)
            uid = distrib_cfg.get('gpg', {}).get('uid', None)
            gpg = GpgInfo(keyfile, keyid, args.passphrase, uid)

            releases = release_list({dist_id: distrib_cfg.get('versions', {})})
            if args.distributions is not None and len(args.distributions) != 0:
                releases = release_filter(
                    releases, {e for el in args.distributions for e in el})

            dist_packages = set()
            for dist_release in releases:
                for name, pkg_cfg in cfg['packages'].items():
                    if not isinstance(pkg_cfg, dict):
                        pkg_cfg = {'value': pkg_cfg}
                    if dist_release in release_list(
                            pkg_cfg.get('disable', {})):
                        continue
                    if 'debian' in dist_release.distrib.id_like:
                        dist_packages.add(
                            DebPackageBuilder(
                                dist_release,
                                base_dir,
                                gpg,
                                name,
                                downloader,
                                pkg_cfg.get('orig', None),
                                1))
                    if 'fedora' in dist_release.distrib.id_like:
                        dist_packages.add(
                            RpmPackageBuilder(
                                dist_release,
                                base_dir,
                                gpg,
                                name,
                                downloader,
                                pkg_cfg.get('orig', None),
                                1))

            output_path = base_dir / \
                distrib_cfg.get('repository', {}).get('path', 'repositories')
            url = distrib_cfg.get('repository', {}).get('url')
            name = distrib_cfg.get('repository', {}).get('name')
            description = distrib_cfg.get('repository', {}).get('description')
            if 'debian' in next(distributions(id=dist_id)).id_like:
                repo_builder = DebRepoBuilder(
                    output_path, gpg, url, name, description, base_dir / 'README.tpl')
            elif 'fedora' in next(distributions(id=dist_id)).id_like:
                repo_builder = RpmRepoBuilder(
                    output_path, gpg, url, name, description, base_dir / 'README.tpl')

            repo_builders.append((repo_builder, dist_packages, ))
            packages.update(dist_packages)

        build_packages(packages, parallelism=args.jobs)
        for builder in repo_builders:
            builder[0].build(builder[1])

    except KeyboardInterrupt as err:
        if args.interactive:
            input("Press Enter to clean-up and continue...")
            raise
        exit(1)
    except CalledProcessError as err:
        print("\033[31m[ERROR]\033[39m   * Error while running subprocess:\n$ " +
              ' '.join([str(ce) for ce in err.cmd]), file=stderr)
        if err.stdout is not None:
            print(err.stdout)
        if err.stderr is not None:
            print(err.stderr, file=stderr)
        print(file=stderr)
        if args.interactive:
            input("Press Enter to clean-up and quit...")
            raise
        exit(2)
    except Exception as err:
        print("\033[31m[ERROR]\033[39m   * " + str(err))
        if args.interactive:
            input("Press Enter to clean-up and continue...")
            raise
        exit(3)


if __name__ == '__main__':
    main()