Skip to content

Commit

Permalink
Merge pull request #22 from vondrt4/openstack-work
Browse files Browse the repository at this point in the history
Upgrade OpenStack support from AMI compatibility to qcow2 native
  • Loading branch information
JedMeister authored Mar 31, 2017
2 parents b53310b + af2f2d9 commit 124fcd4
Show file tree
Hide file tree
Showing 11 changed files with 459 additions and 42 deletions.
82 changes: 67 additions & 15 deletions bin/openstack-bundle
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ rootfs=$1
name=$(echo $rootfs | sed 's/.rootfs//')
appname=$(echo $name |sed 's/turnkey-\(.*\)-[0-9].*/\1/')



case "$appname" in
canvas) loopsize_padding=786432 ;;
ejabberd) loopsize_padding=524288 ;;
Expand All @@ -46,29 +48,79 @@ loopsize=$[$rootsize + $loopsize_padding]

info "creating sparse loopback"
dd if=/dev/null of=$rootfs.img bs=1 seek=${loopsize}K
mkfs.ext4 -F -j $rootfs.img

mkdir $rootfs.img.mount
mount -o loop $rootfs.img $rootfs.img.mount
info "creating partition"
#inspired by package openstack-debian-images
PARTED=/sbin/parted
AMI_NAME=$rootfs.img
${PARTED} -s ${AMI_NAME} mktable msdos
${PARTED} -s -a optimal ${AMI_NAME} mkpart primary ext3 1Mi 100%
${PARTED} -s ${AMI_NAME} set 1 boot on
install-mbr ${AMI_NAME}
RESULT_KPARTX=`kpartx -asv ${AMI_NAME} 2>&1`

if echo "${RESULT_KPARTX}" | grep "^add map" ; then
LOOP_DEVICE=`echo ${RESULT_KPARTX} | cut -d" " -f3`
info "kpartx mounted using: ${LOOP_DEVICE}"
else
fatal "It seems kpartx didn't mount the image correctly: exiting."
fi

cleanup(){
error=$?
[ ! -d "${MOUNT_DIR}" ] && return
echo
echo "error $error, umounting $MOUNT_DIR"
chroot ${MOUNT_DIR} umount /proc || true
chroot ${MOUNT_DIR} umount /sys || true
umount ${MOUNT_DIR}
rmdir ${MOUNT_DIR}
kpartx -d ${AMI_NAME}
exit $error
}
trap "cleanup" EXIT TERM INT

mkfs.ext4 -F -j /dev/mapper/${LOOP_DEVICE}
# No fsck because of X days without checks
tune2fs -i 0 /dev/mapper/${LOOP_DEVICE}

MOUNT_DIR=`mktemp -d -t build-debimg.XXXXXX`
mount -o loop /dev/mapper/${LOOP_DEVICE} ${MOUNT_DIR}

info "syncing rootfs to loopback"
rsync -a -t -r -S -I -H $rootfs/ $rootfs.img.mount
rsync -a -t -r -S -I -H $rootfs/ ${MOUNT_DIR}

info "umount loopback"
umount -d $rootfs.img.mount
rmdir $rootfs.img.mount
info "install extlinux"
mkdir -p ${MOUNT_DIR}/boot/extlinux
echo "default linux
timeout 1
label linux
kernel /vmlinuz
append initrd=/initrd.img root=/dev/vda1 biosdevname=0 net.ifnames=0 console=tty0 console=ttyS0,115200 ro" > ${MOUNT_DIR}/boot/extlinux/extlinux.conf
extlinux --install ${MOUNT_DIR}/boot/extlinux

info "setting up image directory"
mkdir $name
mv $rootfs.img $name/$name.img
cp $rootfs/boot/vmlinuz-* $name/$name-kernel
cp $rootfs/boot/initrd.img-* $name/$name-initrd
info "umount loopback"
umount -d ${MOUNT_DIR}
rmdir ${MOUNT_DIR}

fsck.ext3 -f /dev/mapper/${LOOP_DEVICE} || true

#the next command failed once, so
sync
kpartx -d ${AMI_NAME}

info "creating qcow2 image"
QCOW2_NAME=$name-openstack.qcow2
QEMU_VERSION=`qemu-img --help | head -n 1 | cut -d" " -f3 | cut -d"," -f1`
if dpkg --compare-versions ${QEMU_VERSION} gt 1.0 ; then
OTHER_QEMU_IMG_OPTIONS=" -o compat=0.10"
else
OTHER_QEMU_IMG_OPTIONS=""
fi

info "creating $name-openstack.tar.gz"
tar --sparse -zcvf $name-openstack.tar.gz $name
qemu-img convert -c -f raw ${AMI_NAME}${OTHER_QEMU_IMG_OPTIONS} -O qcow2 ${QCOW2_NAME}

if [ -z "$BT_DEBUG" ]; then
info "removing directory"
rm -rf $name
fi

74 changes: 74 additions & 0 deletions bin/openstack-bundle-ami
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/bin/bash -e
# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org
#
# This file is part of buildtasks.
#
# Buildtasks is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.


fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; }
info() { echo "INFO [$(basename $0)]: $@"; }

usage() {
cat<<EOF
Syntax: $0 rootfs
Bundles rootfs into an openstack tarball
Arguments::
rootfs - root filesystem path
EOF
exit 1
}

if [[ "$#" != "1" ]]; then
usage
fi

rootfs=$1
name=$(echo $rootfs | sed 's/.rootfs//')
appname=$(echo $name |sed 's/turnkey-\(.*\)-[0-9].*/\1/')

case "$appname" in
canvas) loopsize_padding=524288 ;;
ejabberd) loopsize_padding=524288 ;;
appengine-python) loopsize_padding=524288 ;;
*) loopsize_padding=262144 ;;
esac

info "getting size for loopback"
rootsize=$(du -s $rootfs | awk '{print $1}')
loopsize=$[$rootsize + $loopsize_padding]

info "creating sparse loopback"
dd if=/dev/null of=$rootfs.img bs=1 seek=${loopsize}K
mkfs.ext4 -F -j $rootfs.img

mkdir $rootfs.img.mount
mount -o loop $rootfs.img $rootfs.img.mount

info "syncing rootfs to loopback"
rsync -a -t -r -S -I -H $rootfs/ $rootfs.img.mount

info "umount loopback"
umount -d $rootfs.img.mount
rmdir $rootfs.img.mount

info "setting up image directory"
mkdir $name
mv $rootfs.img $name/$name.img
cp $rootfs/boot/vmlinuz-* $name/$name-kernel
cp $rootfs/boot/initrd.img-* $name/$name-initrd

info "creating $name-openstack.tar.gz"
tar --sparse -zcvf $name-openstack.tar.gz $name

if [ -z "$BT_DEBUG" ]; then
info "removing directory"
rm -rf $name
fi

8 changes: 4 additions & 4 deletions bt-openstack
Original file line number Diff line number Diff line change
Expand Up @@ -86,17 +86,17 @@ $BT/bin/aptconf-tag $rootfs openstack

$BT/bin/openstack-bundle $rootfs

$BT/bin/generate-signature $O/$name-openstack.tar.gz
$BT/bin/generate-signature $O/$name-openstack.qcow2

$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.tar.gz.buildenv
$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.qcow2.buildenv

# publish if specified
if [ "$publish" == "yes" ]; then
export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/
$BT/bin/publish-files $O/$name-openstack.tar.gz
$BT/bin/publish-files $O/$name-openstack.qcow2

export PUBLISH_DEST=${BT_PUBLISH_META}/
$BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv}
$BT/bin/publish-files $O/$name-openstack.{qcow2.sig,qcow2.buildenv}
fi

if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then
Expand Down
93 changes: 93 additions & 0 deletions bt-openstack-ami
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/bin/bash -e
# Copyright (c) 2011-2015 TurnKey GNU/Linux - http://www.turnkeylinux.org
#
# This file is part of buildtasks.
#
# Buildtasks is free software; you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by the
# Free Software Foundation; either version 3 of the License, or (at your
# option) any later version.


fatal() { echo "FATAL [$(basename $0)]: $@" 1>&2; exit 1; }
warning() { echo "WARNING [$(basename $0)]: $@"; }
info() { echo "INFO [$(basename $0)]: $@"; }

usage() {
cat<<EOF
Syntax: $(basename $0) [--publish] appname
Converts appliance appname (e.g., core) to openstack image
Options::
--publish - if set, image will be devpay'ed and made public
Environment::
BT_DEBUG - turn on debugging
EOF
exit 1
}

while [ "$1" != "" ]; do
case $1 in
--help|-h ) usage;;
--publish) publish="yes";;
*) if [ -n "$appname" ]; then usage; else appname=$1; fi ;;
esac
shift
done

[ -n "$appname" ] || usage
[ -n "$publish" ] || warning "--publish was not specified"

[ -n "$BT_DEBUG" ] && set -x

export BT=$(dirname $(readlink -f $0))
export BT_CONFIG=$BT/config
. $BT_CONFIG/common.cfg
. $BT_CONFIG/build.cfg

O=$BT_BUILDS/openstack
mkdir -p $O

[ -n "$BT_VERSION" ] || fatal "BT_VERSION not set"

isofile=turnkey-$appname-$BT_VERSION.iso
name=turnkey-$appname-$BT_VERSION
rootfs=$name.rootfs
cdroot=$name.cdroot

$BT/bin/iso-download $BT_ISOS $BT_VERSION $appname
$BT/bin/iso-verify $BT_ISOS $BT_VERSION $appname

cd $O
tklpatch-extract-iso $BT_ISOS/$isofile

tklpatch-apply $rootfs $BT/patches/headless
tklpatch-apply $rootfs $BT/patches/cloud
tklpatch-apply $rootfs $BT/patches/openstack-ami
$BT/bin/rootfs-cleanup $rootfs

$BT/bin/aptconf-tag $rootfs openstack

$BT/bin/openstack-bundle-ami $rootfs

$BT/bin/generate-signature $O/$name-openstack.tar.gz

$BT/bin/generate-buildenv openstack $BT_ISOS/$isofile.sig > $O/$name-openstack.tar.gz.buildenv

# publish if specified
if [ "$publish" == "yes" ]; then
export PUBLISH_DEST=${BT_PUBLISH_IMGS}/openstack/
$BT/bin/publish-files $O/$name-openstack.tar.gz

export PUBLISH_DEST=${BT_PUBLISH_META}/
$BT/bin/publish-files $O/$name-openstack.{tar.gz.sig,tar.gz.buildenv}
fi

if [ -z "$BT_DEBUG" ] && ! (mount | grep -q $(basename $rootfs)); then
rm -rf $rootfs
rm -rf $cdroot
fi

5 changes: 5 additions & 0 deletions docs/setup
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ bt-ec2 ::
apt-get install parted ec2metadata
pip install boto

bt-openstack ::

apt-get install parted ec2metadata mbr qemu kpartx extlinux
pip install boto

bt-vm ::

# vmware ovftool
Expand Down
36 changes: 36 additions & 0 deletions patches/openstack-ami/conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/bash -ex

install() {
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get -y \
-o DPkg::Options::=--force-confdef \
-o DPkg::Options::=--force-confold \
install $@
}

# install useful packages
install ebsmount sysvinit-core systemd-shim

# remove systemd (sysvinit used in container)
dpkg --purge systemd-sysv systemd || true

# support hot-plugging of attached volumes
echo "acpiphp" >> /etc/modules

# hold kernel (not used in image, pro-longs sec-updates)
ARCH=$(dpkg --print-architecture)
case "$ARCH" in
"i386")
META_KERNEL="linux-image-686";
;;
"amd64")
META_KERNEL="linux-image-amd64";
;;
*)
fatal "non-supported architecture: $ARCH";
;;
esac
KERNEL=$(echo /boot/vmlinuz-* | sed 's|/boot/vmlinuz-|linux-image-|')
echo "$KERNEL hold" | dpkg --set-selections
echo "$META_KERNEL hold" | dpkg --set-selections

6 changes: 6 additions & 0 deletions patches/openstack-ami/overlay/etc/fstab
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# /etc/fstab: static file system information.
# <file system> <mount point> <type> <options> <dump> <pass>
proc /proc proc nodev,noexec,nosuid 0 0
/dev/vda / ext4 defaults 0 0
/dev/vdb /mnt auto defaults 0 0

29 changes: 8 additions & 21 deletions patches/openstack/conf
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,16 @@ install() {
install $@
}

#conflicts with cloud-utils, which is a dependency of cloud-initramfs-growroot
dpkg --purge ec2metadata

# install useful packages
install ebsmount sysvinit-core systemd-shim
install ebsmount cloud-initramfs-growroot extlinux

# remove systemd (sysvinit used in container)
dpkg --purge systemd-sysv systemd || true
#on the other hand, inithooks needs it and the other implementation is written by Alon Swartz anyway
DEBIAN_FRONTEND=noninteractive apt-get -y -o DPkg::Options::=--force-confdef \
-o DPkg::Options::=--force-confold -o DPkg::Options::=--force-overwrite \
install ec2metadata

# support hot-plugging of attached volumes
echo "acpiphp" >> /etc/modules

# hold kernel (not used in image, pro-longs sec-updates)
ARCH=$(dpkg --print-architecture)
case "$ARCH" in
"i386")
META_KERNEL="linux-image-686";
;;
"amd64")
META_KERNEL="linux-image-amd64";
;;
*)
fatal "non-supported architecture: $ARCH";
;;
esac
KERNEL=$(echo /boot/vmlinuz-* | sed 's|/boot/vmlinuz-|linux-image-|')
echo "$KERNEL hold" | dpkg --set-selections
echo "$META_KERNEL hold" | dpkg --set-selections

3 changes: 1 addition & 2 deletions patches/openstack/overlay/etc/fstab
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# /etc/fstab: static file system information.
# <file system> <mount point> <type> <options> <dump> <pass>
proc /proc proc nodev,noexec,nosuid 0 0
/dev/vda / ext4 defaults 0 0
/dev/vdb /mnt auto defaults 0 0
/dev/vda1 / ext4 defaults 0 0

Loading

0 comments on commit 124fcd4

Please sign in to comment.