Skip to content

Commit

Permalink
Reuse image tables and methods for container builds
Browse files Browse the repository at this point in the history 
This removes some db schema which was introduced in last release. I
don't have proper migration plan so I hope nobody have it in the
database.
  • Loading branch information
@pbabinca
pbabinca committed Jun 8, 2015
1 parent 498501b commit b16e16d
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 274 deletions.
24 changes: 0 additions & 24 deletions docs/schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,3 @@
-- still needs work

INSERT INTO channels (name) VALUES ('container');

DROP TABLE IF EXISTS container_builds;
DROP TABLE IF EXISTS container_archives;
DROP TABLE IF EXISTS container_listing;

-- track container builds
CREATE TABLE container_builds (
build_id INTEGER NOT NULL PRIMARY KEY REFERENCES build(id)
) WITHOUT OIDS;

INSERT INTO archivetypes (name, description, extensions) values ('container', 'Container image', 'tar.gz');

CREATE TABLE container_archives (
archive_id INTEGER NOT NULL PRIMARY KEY REFERENCES archiveinfo(id),
arch VARCHAR(16) NOT NULL
) WITHOUT OIDS;

-- tracks the contents of an container
CREATE TABLE container_listing (
container_id INTEGER NOT NULL REFERENCES container_archives(archive_id),
rpm_id INTEGER NOT NULL REFERENCES rpminfo(id),
UNIQUE (container_id, rpm_id)
) WITHOUT OIDS;
CREATE INDEX container_listing_rpms on container_listing(rpm_id);
18 changes: 9 additions & 9 deletions koji_containerbuild/plugins/builder_containerbuild.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,11 +229,11 @@ def initContainerBuild(self, name, version, release, target_info, opts):
raise koji.BuildError("package (container) %s not in list for tag %s" % (name, target_info['dest_tag_name']))
elif pkg_cfg['blocked']:
raise koji.BuildError("package (container) %s is blocked for tag %s" % (name, target_info['dest_tag_name']))
return self.session.host.initContainerBuild(self.id,
return self.session.host.initImageBuild(self.id,
dict(name=name,
version=version,
release=release,
epoch=0))
version=version,
release=release,
epoch=0))

def runBuilds(self, src, target_info, arches, scratch=False):
subtasks = {}
Expand Down Expand Up @@ -335,12 +335,12 @@ def handler(self, src, target, opts=None):
self._raise_if_image_failed(result['osbs_build_id'])
if opts.get('scratch'):
# scratch builds do not get imported
self.session.host.moveContainerToScratch(self.id,
results_xmlrpc)
self.session.host.moveImageBuildToScratch(self.id,
results_xmlrpc)
else:
self.session.host.completeContainerBuild(self.id,
bld_info['id'],
results_xmlrpc)
self.session.host.completeImageBuild(self.id,
bld_info['id'],
results_xmlrpc)
except (SystemExit, ServerExit, KeyboardInterrupt):
# we do not trap these
raise
Expand Down
242 changes: 1 addition & 241 deletions koji_containerbuild/plugins/hub_containerbuild.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,15 @@

import koji
from koji.context import context
from koji.plugin import export, export_in
from koji.plugin import export

sys.path.insert(0, '/usr/share/koji-hub/')
import kojihub
#kojihub = inspect.getmodule(sys._getframe(2)).kojihub

#logger = logging.getLogger('koji.plugins.containerbuild')
logger = logging.getLogger('koji.plugins')


# TODO: This monkey patch won't work for other koji calls (e.g. from CLI)
def pathinfo_containerbuild(self, build):
"""Return the directory where the containers for the build are stored"""
return self.build(build) + '/containers'

koji.pathinfo.containerbuild = pathinfo_containerbuild


@export
def buildContainer(src, target, opts=None, priority=None, channel='container'):
"""Create a container build task
Expand All @@ -71,234 +62,3 @@ def buildContainer(src, target, opts=None, priority=None, channel='container'):
if channel:
taskOpts['channel'] = channel
return kojihub.make_task('buildContainer', [src, target, opts], **taskOpts)


@export_in(module='host')
def moveContainerToScratch(task_id, results):
"""move a completed image scratch build into place"""
host = kojihub.Host()
host.verify()
task = kojihub.Task(task_id)
task.assertHost(host.id)

scratchdir = koji.pathinfo.scratch()
logger.debug('scratch image results: %s', results)
for sub_results in results.values():
sub_task_id = sub_results['task_id']
sub_task = kojihub.Task(sub_task_id)
workdir = koji.pathinfo.task(sub_task_id)
username = kojihub.get_user(sub_task.getOwner())['name']
destdir = os.path.join(scratchdir, username, 'task_%s' % task_id)
koji.ensuredir(destdir)
for img in sub_results['logs']:
src = os.path.join(workdir, img)
logger.debug('Source log: %s', img)
# verify files exist
if not os.path.exists(src):
raise koji.GenericError("no such file: %s" % src)
dest = os.path.join(destdir, img)
logger.debug('renaming %s to %s', src, dest)
os.rename(src, dest)
os.symlink(dest, src)


@export_in(module='host')
def initContainerBuild(task_id, build_info):
"""create a new in-progress container build"""
host = kojihub.Host()
host.verify()
task = kojihub.Task(task_id)
task.assertHost(host.id)
data = build_info.copy()
data['task_id'] = task_id
data['owner'] = task.getOwner()
data['state'] = koji.BUILD_STATES['BUILDING']
data['completion_time'] = None
build_id = kojihub.new_build(data)
data['id'] = build_id
new_container_build(data)
return data


def new_container_build(build_info):
"""
Added container metadata to an existing build. This is just the buildid so
that we can distinguish container builds from other types.
"""
# We don't have to worry about updating an container build because the id
# is the only thing we care about, and that should never change if a build
# fails first and succeeds later on a resubmission.
query = kojihub.QueryProcessor(tables=('container_builds',),
columns=('build_id',),
clauses=('build_id = %(build_id)i',),
values={'build_id': build_info['id']})
result = query.executeOne()
if not result:
insert = kojihub.InsertProcessor('container_builds')
insert.set(build_id=build_info['id'])
insert.execute()


@export_in(module='host')
def completeContainerBuild(task_id, build_id, results):
"""Set an container build to the COMPLETE state"""
host = kojihub.Host()
host.verify()
task = kojihub.Task(task_id)
task.assertHost(host.id)
importContainer(task_id, build_id, results)

st_complete = koji.BUILD_STATES['COMPLETE']
update = kojihub.UpdateProcessor('build', clauses=['id=%(build_id)i'],
values={'build_id': build_id})
update.set(id=build_id, state=st_complete)
update.rawset(completion_time='now()')
update.execute()
# send email
kojihub.build_notification(task_id, build_id)


def importContainer(task_id, build_id, results):
"""
Import a built container, populating the database with metadata and
moving the container to its final location.
"""
for sub_results in results.values():
importContainerInternal(task_id, build_id, sub_results)


# TODO: heavily hacked version of kojihub import_archive which accepts empty
# filepath. Do this in some less hackish way but still support workflow of
# later promotion of docker image.
def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
"""
Import an archive file and associate it with a build. The archive can
be any non-rpm filetype supported by Koji or None if the file doesn't exist
(yet) and we need to reference it.
filepath: full path to the archive file or None
buildinfo: dict of information about the build to associate the archive with (as returned by getBuild())
type: type of the archive being imported. Currently supported archive types: maven, win, image
typeInfo: dict of type-specific information
buildroot_id: the id of the buildroot the archive was built in (may be null)
"""
if filepath and not os.path.exists(filepath):
raise koji.GenericError, 'no such file: %s' % filepath

archiveinfo = {'buildroot_id': buildroot_id}
if filepath:
filename = koji.fixEncoding(os.path.basename(filepath))
archiveinfo['filename'] = filename
archivetype = kojihub.get_archive_type(filename, strict=True)
else:
#XXX: hack, see above
archiveinfo['filename'] = ''
archivetype = kojihub.get_archive_type(type_name='container',
strict=True)
archiveinfo['type_id'] = archivetype['id']
archiveinfo['build_id'] = buildinfo['id']
if filepath:
archiveinfo['size'] = os.path.getsize(filepath)
else:
#XXX: hack, see above
archiveinfo['size'] = 0

if filepath:
archivefp = file(filepath)
m = koji.util.md5_constructor()
while True:
contents = archivefp.read(8192)
if not contents:
break
m.update(contents)
archivefp.close()
archiveinfo['checksum'] = m.hexdigest()
archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES['md5']
else:
#XXX: hack, see above
archiveinfo['checksum'] = ''
archiveinfo['checksum_type'] = 0

koji.plugin.run_callbacks('preImport', type='archive', archive=archiveinfo,
build=buildinfo, build_type=type,
filepath=filepath)

# XXX verify that the buildroot is associated with a task that's associated with the build
archive_id = kojihub._singleValue("SELECT nextval('archiveinfo_id_seq')",
strict=True)
archiveinfo['id'] = archive_id
insert = kojihub.InsertProcessor('archiveinfo', data=archiveinfo)
insert.execute()

if type == 'container':
insert = kojihub.InsertProcessor('container_archives')
insert.set(archive_id=archive_id)
insert.set(arch=typeInfo['arch'])
insert.execute()
# TODO this will be used if we really have file name
if filepath:
imgdir = os.path.join(koji.pathinfo.containerbuild(buildinfo))
kojihub._import_archive_file(filepath, imgdir)
# import log files?
else:
raise koji.BuildError, 'unsupported archive type: %s' % type

archiveinfo = kojihub.get_archive(archive_id, strict=True)
koji.plugin.run_callbacks('postImport', type='archive',
archive=archiveinfo, build=buildinfo,
build_type=type, filepath=filepath)
return archiveinfo


def importContainerInternal(task_id, build_id, containerdata):
"""
Import container info and the listing into the database, and move an
container to the final resting place. The filesize may be reported as a
string if it exceeds the 32-bit signed integer limit. This function will
convert it if need be. This is the completeBuild for containers; it should
not be called for scratch container builds.
containerdata is:
arch - the arch if the container
files - files associated with the container
rpmlist - the list of RPM NVRs installed into the container
"""
host = kojihub.Host()
host.verify()
task = kojihub.Task(task_id)
task.assertHost(host.id)

koji.plugin.run_callbacks('preImport', type='container',
container=containerdata)

# import the build output
build_info = kojihub.get_build(build_id, strict=True)
containerdata['relpath'] = koji.pathinfo.taskrelpath(containerdata['task_id'])
archives = []
archives.append(import_archive(None, build_info, 'container',
containerdata))

# record all of the RPMs installed in the containers
# verify they were built in Koji or in an external repo
rpm_ids = []
for an_rpm in containerdata['rpmlist']:
location = an_rpm.get('location')
if location:
data = kojihub.add_external_rpm(an_rpm, location, strict=False)
else:
data = kojihub.get_rpm(an_rpm, strict=True)
rpm_ids.append(data['id'])

# associate those RPMs with the container
q = """INSERT INTO container_listing (container_id,rpm_id)
VALUES (%(container_id)i,%(rpm_id)i)"""
for archive in archives:
logger.info('working on archive %s' % archive)
if archive['filename'].endswith('xml'):
continue
logger.info('associating installed rpms with %s' % archive['id'])
for rpm_id in rpm_ids:
kojihub._dml(q, {'container_id': archive['id'], 'rpm_id': rpm_id})

koji.plugin.run_callbacks('postImport', type='container',
container=containerdata, fullpath=None)

0 comments on commit b16e16d

Please sign in to comment.