Cleaned up dcarley's EBS work, adding documentation

.gitignore

@@ -1,3 +1,4 @@
 *.egg-info
 dist
 build
+work/

README.md

@@ -0,0 +1,56 @@
+# ami-creator
+
+A simple tool built on python-imgcreate to create CentOS AMIs for EC2 using
+Kickstart configs.  Supports ebs- and instance-store-backed instances.
+
+## howto
+
+1. clone this repo on an EC2 instance
+2. attach a 10G EBS volume, attach it so it appears at `/dev/xvdj`
+3. create `/media/ephemeral0/build` and `/media/ephemeral0/cache` directories
+4. create the base image from a kickstart config:
+
+    sudo ./ami_creator/ami_creator.py -c ks-centos6.cfg -n my-image-name -t /media/ephemeral0/build --cache=/media/ephemeral0/cache
+
+5. transfer the image to the attached EBS volume:
+
+    sudo prepare-volume.sh
+
+6. create and launch the instance:
+
+    ./create-image.sh
+
+
+## add'l resources
+
+* [Announcing ami-creator](http://velohacker.com/2010/12/13/announcing-ami-creator/)
+* [CentOS AMIs from Kickstart](http://dan.carley.co/blog/2012/04/17/centos-amis-from-kickstart/)
+* [How to build your own S3- and EBS-backed instances](http://amazonaws.michael--martinez.com)
+* [Using Your Own Linux Kernels - Amazon Elastic Compute Cloud](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html)
+* [How to make your own CentOS 6 AMIs](http://www.bashton.com/blog/2012/how-to-make-your-own-centos-6-amis/)
+* [CentOS 5 kickstart options](http://www.centos.org/docs/5/html/Installation_Guide-en-US/s1-kickstart2-options.html)
+
+## original `README`
+
+    ami-creator
+
+    A simple tool based on python-imgcreate to create Fedora/Red Hat style 
+    images that can be used as an AMI in EC2.
+
+    Takes a kickstart config like the rest of livecd-creator, etc and spits out a
+    disk image file that's suitable to upload as an s3 ami.  To do the upload right
+    now, you'll want to run something like
+        ec2-bundle-image
+        ec2-upload-bundle
+        ec2-register
+    after having created your base image file.  
+
+    Tested with the following as guests:
+      * CentOS 5.5
+      * Fedora 14
+    See the configs/ directory for example configs that work for each of these.
+
+
+    Jeremy Katz <[email protected]>
+    2010 December 10
+

ami_creator/ami_creator.py

@@ -25,8 +25,6 @@
 
 import imgcreate
 
-import rpmUtils.arch
-
 class Usage(Exception):
     def __init__(self, msg = None):
         Exception.__init__(self, msg)
@@ -81,27 +79,39 @@ def __init__(self, *args, **kwargs):
                           "sd_mod", "mptsas", "sg" ]
         self.__modules.extend(imgcreate.kickstart.get_modules(self.ks))
 
+        ## for some reason ExtDiskMount prefixes the fslabel with "_"
+        self.__real_fslabel = "_" + self.fslabel
+
     # FIXME: refactor into imgcreate.LoopImageCreator
     def _get_kernel_options(self):
         """Return a kernel options string for bootloader configuration."""
         r = imgcreate.kickstart.get_kernel_args(self.ks, default = "ro")
         return r
 
     def _get_fstab(self):
-        s = "LABEL=_/   /        %s      defaults         0 0\n" % self._fstype
+        s = "LABEL=%s   /        %s      defaults         0 0\n" % (self.__real_fslabel, self._fstype)
 
         s += self._get_fstab_special()
         return s
 
     def _create_bootconfig(self):
-        imgtemplate = """title %(title)s %(version)s
-        root (hd0)
-        kernel /boot/vmlinuz-%(version)s root=LABEL=_/ %(bootargs)s
-        initrd /boot/%(initrdfn)s-%(version)s.img
+        imgtemplate = """
+title %(title)s %(version)s EBS
+    root (hd0,0)
+    kernel /boot/vmlinuz-%(version)s root=LABEL=%(fslabel)s %(bootargs)s
+    initrd /boot/%(initrdfn)s-%(version)s.img
+
+title %(title)s %(version)s S3
+    root (hd0)
+    kernel /boot/vmlinuz-%(version)s root=LABEL=%(fslabel)s %(bootargs)s
+    initrd /boot/%(initrdfn)s-%(version)s.img
 """
 
-        cfg = """default=0
+        cfg = """
+default=0
+fallback=1
 timeout=%(timeout)s
+hiddenmenu
 
 """ % { "timeout": imgcreate.kickstart.get_timeout(self.ks, 5) }
 
@@ -119,7 +129,8 @@ def _create_bootconfig(self):
             cfg += imgtemplate % {"title": self.name,
                                   "version": version,
                                   "initrdfn": initrdfn,
-                                  "bootargs": self._get_kernel_options()}
+                                  "bootargs": self._get_kernel_options(),
+                                  "fslabel": self.__real_fslabel}
 
         with open(self._instroot + "/boot/grub/grub.conf", "w") as grubcfg:
             grubcfg.write(cfg)
@@ -193,7 +204,7 @@ def main():
     if options.name:
         name = options.name
 
-    creator = AmiCreator(ks, name, "/")
+    creator = AmiCreator(ks, name, fslabel="root")
     creator.tmpdir = os.path.abspath(options.tmpdir)
     if options.cachedir:
         options.cachedir = os.path.abspath(options.cachedir)

ks-centos6.cfg

@@ -1,31 +1,45 @@
-# Build a basic CentOS 6 AMI
 lang en_US.UTF-8
 keyboard us
-timezone US/Eastern
-auth --useshadow --enablemd5
-selinux --permissive
-firewall --enabled --ssh
-bootloader --timeout=1  --append xen_blkfront.sda_is_xvda=1
+timezone --utc UTC
+authconfig --enableshadow --passalgo=sha512
+selinux --disabled
+firewall --disabled
+firstboot --disabled
+
+## selinux=0 required to log in via ssh; something missing in this config, I'm
+## sure, but I hate/don't understand selinux, so disabling with a vengance.
+## modifying /etc/sysconfig/selinux in %post below *should* do the trick, but it
+## isn't.  punting.
+
+## "xen_blkfront.sda_is_xvda=1" -- https://bugzilla.redhat.com/show_bug.cgi?id=729586
+## remainder stolen from bashton
+## dracut options: http://fedoraproject.org/wiki/Dracut/Options
+
+bootloader --timeout=1 --append="xen_blkfront.sda_is_xvda=1 selinux=0 rd_NO_LUKS rd_NO_LVM rd_NO_DM rd_NO_MD LANG=en_US.UTF-8 KEYTABLE=us SYSFONT=latarcyrheb-sun16 console=hvc0 crashkernel=auto"
+
 network --bootproto=dhcp --device=eth0 --onboot=on
-services --enabled=network
+services --enabled=network,sshd,cloud-init
 
 
 # Uncomment the next line to make the root password be password
 # By default the root password is locked
 #rootpw password
-rootpw --lock
+#rootpw --lock
 
 #
 # Define how large you want your rootfs to be
 # NOTE: S3-backed AMIs have a limit of 10G
 #
+# fs label is not used for the root fs! if you need to refer to the label, use
+# "_root".
 part / --size 768 --fstype ext4
 
 #
 # Repositories
 repo --name=CentOS6-Base --mirrorlist=http://mirrorlist.centos.org/?release=6&arch=$basearch&repo=os
 repo --name=CentOS6-Updates --mirrorlist=http://mirrorlist.centos.org/?release=6&arch=$basearch&repo=updates
-repo --name=EPEL --baseurl=http://download.fedoraproject.org/pub/epel/6/$basearch/
+#repo --name=EPEL --baseurl=http://download.fedoraproject.org/pub/epel/6/$basearch/
+repo --name=EPEL --baseurl=http://mirror.pnl.gov/epel/6/$basearch/
 
 
 #
@@ -34,8 +48,6 @@ repo --name=EPEL --baseurl=http://download.fedoraproject.org/pub/epel/6/$basearc
 #
 %packages --nobase --instLangs=en
 @core
-system-config-securitylevel-tui
-system-config-firewall-base
 audit
 pciutils
 bash
@@ -53,6 +65,7 @@ acpid
 openssh-clients
 openssh-server
 curl
+epel-release
 cloud-init
 sudo
 
@@ -64,16 +77,22 @@ iputils
 
 # set up ec2-user
 %post --erroronfail
+set -e
+set -x
+set -u
+
 /usr/sbin/useradd ec2-user
 echo 'ec2-user ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
 
-%end
+## hrm, even with "selinux --disabled" above, selinux still appears to be enabled!
+## relevant? https://bugzilla.redhat.com/show_bug.cgi?id=690279
+sed -i -e 's#^SELINUX=.*$#SELINUX=disabled#' /etc/sysconfig/selinux
 
+## only one console needed (if that…)
+sed -i -e 's#^ACTIVE_CONSOLES=.*$#ACTIVE_CONSOLES=/dev/tty1#' /etc/sysconfig/init 
 
-#
-# Add custom post scripts after the base post.
-#
-%post
-%end
-
+## this could help diagnostics later
+sed -i -e 's#^BOOTUP=.*$#BOOTUP=verbose#' /etc/sysconfig/init 
 
+yum clean all
+%end

utils/BUILDING_IN_DOCKER.md

@@ -0,0 +1,32 @@
+Since `ami_creator` uses the yum on the host system to build an image, you can
+run into problems if the host and target distributions don't agree.  For
+example, CentOS 7 packages use xz compression, but that's not available to yum
+on CentOS 6.  And it seems plausible that building a CentOS 6 image on CentOS 7
+would result in an RPM database that can't be read by a live CentOS 6 instance.
+
+Fortunately, we have Docker.
+
+## `build_in_docker.sh`
+
+Usage:
+
+    build_in_docker.sh <host distro> <kickstart config> <name>
+
+`host distro` is matched against the corresponding `Dockerfile.?` in this
+directory.  `../work/build/<name>.img` is created, which can then be used for an
+S3-backed AMI, or written to an EBS volume to be snapshotted and registered as
+an EBS-backed AMI.
+
+## CentOS 6
+
+Build CentOS 6 images using a CentOS 6 host environment.
+
+    ./build_in_docker.sh centos6 /path/to/centos6.ks centos6
+
+## CentOS 7
+
+Build CentOS 7 images using a _Fedora 20_ host environment.  CentOS 7 doesn't
+have the required `python-imgcreate` package available, either in the base
+repository or in EPEL.
+
+    ./build_in_docker.sh fedora20 /path/to/centos7.ks centos7

utils/Dockerfile.centos6

@@ -0,0 +1,8 @@
+# -*- Dockerfile -*-
+FROM centos:centos6
+
+RUN yum -y localinstall http://mirror.pnl.gov/epel/6/i386/epel-release-6-8.noarch.rpm
+RUN yum -y install python-imgcreate
+
+VOLUME [ "/srv/work" ]
+WORKDIR /tmp

utils/Dockerfile.fedora20

@@ -0,0 +1,11 @@
+# -*- Dockerfile -*-
+
+## "xz compression not available" from yum building on centos6
+## python-imgcreate not available on centos7
+## fedora:20 is the current stable version, and python-imgcreate *is* available
+FROM fedora:20
+
+RUN yum -y install python-imgcreate
+
+VOLUME [ "/srv/work" ]
+WORKDIR /tmp

utils/build_in_docker.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -e -u
+
+basedir="$( cd $( dirname $0 )/.. && /bin/pwd -P )"
+workdir="${basedir}/work"
+
+if [ $# -ne 3 ]; then
+    echo "usage: $0 <host_distro> <kickstart_cfg> <name>"
+    exit 1
+fi
+
+host_distro="$1"
+kickstart_cfg="$2"
+name="$3"
+
+docker_file="${basedir}/utils/Dockerfile.${host_distro}"
+docker_image="ami_creator/${host_distro}"
+
+if [ ! -e "${docker_file}" ]; then
+    echo "unsupported host distro: ${host_distro}"
+    exit 1
+fi
+
+if [ ! -e "${kickstart_cfg}" ]; then
+    echo "${kickstart_cfg} does not exist"
+    exit 1
+fi
+
+kickstart_base="$( basename ${kickstart_cfg} )"
+kickstart_root="$( cd $( dirname ${kickstart_cfg} ) && /bin/pwd -P )"
+
+mkdir -p ${workdir}/{build,cache}
+
+## build the image if it doesn't already exist
+docker inspect ${docker_image} >/dev/null 2>&1 || \
+    docker build -t ${docker_image} - < ${docker_file}
+
+docker run \
+    -i -t \
+    --rm \
+    --privileged \
+    --volume=${basedir}:/srv/ami-creator:ro \
+    --volume=${kickstart_root}:/srv/image-config:ro \
+    --volume=${workdir}:/srv/work \
+    ${docker_image} \
+    \
+    /bin/bash -c " \
+        /srv/ami-creator/ami_creator/ami_creator.py \
+            -c /srv/image-config/${kickstart_base} \
+            -n ${name} \
+            -t /tmp \
+            --cache=/srv/work/cache \
+        && mv ${name}.img /srv/work/build/
+    "