git_buildpackage

#!/usr/bin/env python
#
# (C) 2014 by Jan Blunck <jblunck@infradead.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
# See http://www.gnu.org/licenses/gpl-2.0.html for full license text.

import argparse
import datetime
import os
import shutil
import re
import fnmatch
import sys
import tarfile
import subprocess
import atexit
import hashlib
import tempfile
import logging
import glob
import ConfigParser
import StringIO


CLEANUP_DIRS = []


def cleanup(dirs):
    '''Cleaning temporary directories.'''

    logging.info("Cleaning: %s", ' '.join(dirs))

    for d in dirs:
        if not os.path.exists(d):
            continue
        shutil.rmtree(d)


def safe_run(cmd, cwd, interactive=False):
    """Execute the command cmd in the working directory cwd and check return
    value. If the command returns non-zero raise a SystemExit exception."""

    logging.debug("COMMAND: %s", cmd)

    # Ensure we get predictable results when parsing the output of commands
    # like 'git branch'
    env = os.environ.copy()
    env['LANG'] = 'C'

    proc = subprocess.Popen(cmd,
                            shell=False,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT,
                            cwd=cwd,
                            env=env)
    output = ''
    if interactive:
        stdout_lines = []
        while proc.poll() is None:
            for line in proc.stdout:
                print line.rstrip()
                stdout_lines.append(line.rstrip())
        output = '\n'.join(stdout_lines)
    else:
        output = proc.communicate()[0]

    logging.debug("RESULT(%d): %s", proc.returncode, repr(output))

    if proc.returncode:
        sys.exit("ERROR(%d): %s:\n%s" % (proc.returncode, ' '.join(cmd), repr(output)))

    return (proc.returncode, output)


def switch_revision(clone_dir, revision):
    """Switch sources to revision. The GIT revision may refer to any of the
    following:
    - explicit SHA1: a1b2c3d4....
    - the SHA1 must be reachable from a default clone/fetch (generally, must be
      reachable from some branch or tag on the remote).
    - short branch name: "master", "devel" etc.
    - explicit ref: refs/heads/master, refs/tags/v1.2.3,
      refs/changes/49/11249/1
    """

    if revision is None:
        revision = 'master'

    revs = [x + revision for x in ['origin/', '']]
    for rev in revs:
        try:
            safe_run(['git', 'rev-parse', '--verify', '--quiet', rev],
                     cwd=clone_dir)
            text = safe_run(['git', 'reset', '--hard', rev], cwd=clone_dir)[1]
            revision = rev
            print text.rstrip()
            break
        except SystemExit:
            continue
    else:
        sys.exit('%s: No such revision' % revision)

    return revision


def fetch_upstream(url, revision, out_dir):
    """Fetch sources from repository and checkout given revision."""

    # calc_dir_to_clone_to
    basename = os.path.basename(re.sub(r'/.git$', '', url))
    clone_dir = os.path.abspath(os.path.join(out_dir, basename))

    if not os.path.isdir(clone_dir):
        # initial clone
        os.mkdir(clone_dir)

        safe_run(['git', 'clone', '--no-checkout', url, clone_dir],
                 cwd=out_dir, interactive=sys.stdout.isatty())
    else:
        logging.info("Detected cached repository...")
#        UPDATE_CACHE_COMMANDS[scm](url, clone_dir, revision)

    return clone_dir

def sanitize_build_args(build_args):
    """
    Prevent potentially dangerous arguments from being passed to gbp, e.g.
    via cleaner, postexport or other hooks.
    """

    safe_args = re.compile('--git-verbose|--git-upstream-tree=.*|--git-no-pristine-tar')
    p = re.compile('--git-.*|--hook-.*|--.*-hook=.*')

    gbp_args  = [ arg for arg in build_args if safe_args.match(arg) ]
    dpkg_args = [ arg for arg in build_args if not p.match(arg) ]

    ignored_args = list(set(build_args) - set(gbp_args + dpkg_args))
    if ignored_args:
        logging.info("Ignoring build_args: %s" % ignored_args)

    return gbp_args + dpkg_args

def create_source_package(repo_dir, output_dir, revision, build_args, submodules):
    """Create source package via git-buildpackage"""

    if not revision:
        revision = 'master'

    command = ['gbp', 'buildpackage', '--git-notify=off', '--git-force-create',
               '--git-cleaner="true"' ]

    # we are not on a proper local branch due to using git-reset but we anyway
    # use the --git-export option
    command.extend(['--git-ignore-branch', "--git-export-dir=%s" % output_dir,
                    "--git-export=%s" % revision])

    # GBP can load submodules without having to run the git command, and will
    # ignore submodules even if loaded manually unless this option is passed.
    if submodules:
        command.extend(['--git-submodules'])

    # create local pristine-tar branch
    try:
        safe_run(['git', 'rev-parse', '--verify', '--quiet',
                  'origin/pristine-tar'], cwd=repo_dir)
        safe_run(['git', 'update-ref', 'refs/heads/pristine-tar',
                  'origin/pristine-tar'], cwd=repo_dir)
        command.append('--git-pristine-tar')
    except SystemExit:
        command.append('--git-no-pristine-tar')

    if build_args:
        command.extend(sanitize_build_args(build_args.split(' ')))

    logging.debug("Running in %s", repo_dir)

    # Since gbp is the "heart" of this service lets force interactive mode
    safe_run(command, cwd=repo_dir, interactive=True)


def copy_source_package(input_dir, output_dir):
    """Copy Debian sources found in input_dir to output_dir."""

    sources = safe_run(['dpkg-scansources', input_dir], cwd=input_dir)[1]

    FILES_PATTERN = re.compile(r'^Files:(.*(?:\n .*)+)', flags=re.MULTILINE)
    for match in FILES_PATTERN.findall(sources):
        logging.info("Files:")
        for line in match.strip().split("\n"):
            fname = line.strip().split(' ')[2]
            logging.info(" %s", fname)
            input_file = os.path.join(input_dir, fname)
            output_file = os.path.join(output_dir, fname)

            if (args.dch_release_update and fnmatch.fnmatch(fname, '*.dsc')):
               with open(input_file, "a") as dsc_file:
                   dsc_file.write("OBS-DCH-RELEASE: 1")

            shutil.copy(input_file, output_file)


if __name__ == '__main__':
    FORMAT = "%(message)s"
    logging.basicConfig(format=FORMAT, stream=sys.stderr, level=logging.INFO)

    parser = argparse.ArgumentParser(description='Git Tarballs')
    parser.add_argument("--config",
                        default='/etc/obs/services/git_buildpackage',
                        help="Specify config file", metavar="FILE")
    args, remaining_argv = parser.parse_known_args()
    defaults = dict({
        'verbose': False
    })
    if os.path.isfile(args.config):
        logging.info("Reading configuration file %s", args.config)
        configfp = StringIO.StringIO()
        configfp.write('[DEFAULT]\n')
        configfp.write(open(args.config).read())
        configfp.seek(0, os.SEEK_SET)
        config = ConfigParser.SafeConfigParser(defaults)
        config.readfp(configfp)
        defaults = dict(config.defaults())

    parser.set_defaults(**defaults)
    parser.add_argument('--url', required=True,
                        help='upstream tarball URL to download')
    parser.add_argument('--revision',
                        help='revision to package')
    parser.add_argument('--submodules', choices=['enable', 'disable'],
                        default='enable',
                        help='Include git submodules in source artefact')
    parser.add_argument('--outdir', required=True,
                        help='osc service parameter that does nothing')
    parser.add_argument('--build_args', type=str,
                        default='-nc -uc -us -S',
                        help='Parameters passed to git-buildpackage')
    parser.add_argument('--pristine-tar',
                        help='osc service parameter that does nothing')
    parser.add_argument('--dch-release-update', choices=['enable', 'disable'],
                        default='disable',
                        help='Append OBS release number')

    parser.add_argument('--verbose', '-v', action='store_true',
                        help='enable verbose output')

    args = parser.parse_args(remaining_argv)

    # force verbose mode in test-mode
    if os.getenv('DEBUG_GIT_BUILDPACKAGE'):
        args.verbose = True

    for arg in args.build_args.split(' '):
        if arg == '--git-verbose':
            args.verbose = True

    if args.verbose:
        logging.getLogger().setLevel(logging.DEBUG)

    if args.dch_release_update == 'enable':
        args.dch_release_update = True
    else:
        args.dch_release_update = False

    if args.submodules == 'enable':
        args.submodules = True
    else:
        args.submodules = False

    #
    # real main
    #

    # force cleaning of our workspace on exit
    atexit.register(cleanup, CLEANUP_DIRS)

    repodir = None

    # create_dirs
    if repodir is None:
        repodir = tempfile.mkdtemp(dir=args.outdir)
        CLEANUP_DIRS.append(repodir)

    # initial_clone
    clone_dir = fetch_upstream(args.url, args.revision, repodir)

    # switch_to_revision
    revision = switch_revision(clone_dir, args.revision)

    # create_source_package
    create_source_package(clone_dir, repodir, revision=revision,
                          build_args=args.build_args,
                          submodules=args.submodules)

    # copy_source_package
    copy_source_package(repodir, args.outdir)