From 55dc919c2261ee567d0cd7d0fe897df976e30683 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Thu, 15 Feb 2024 13:33:01 +0100 Subject: [PATCH 1/8] bib: port to `bootc install to-filesystem` version of images With the latest version of images we now use the `bootc install to-filesystem` feature instead of our own ostree deploy pipeline. This means that most of our customizations need to be disabled and we need to decide if we will port them in some form or update our README to encourage the use of derrived containters for them. --- bib/cmd/bootc-image-builder/image.go | 4 -- bib/cmd/bootc-image-builder/main_test.go | 53 ++++++++++-------------- 2 files changed, 21 insertions(+), 36 deletions(-) diff --git a/bib/cmd/bootc-image-builder/image.go b/bib/cmd/bootc-image-builder/image.go index 09416516..57c8fc81 100644 --- a/bib/cmd/bootc-image-builder/image.go +++ b/bib/cmd/bootc-image-builder/image.go @@ -78,8 +78,6 @@ func manifestForDiskImage(c *ManifestConfig, rng *rand.Rand) (*manifest.Manifest img := image.NewBootcDiskImage(containerSource) img.Users = users.UsersFromBP(customizations.GetUsers()) - img.Groups = users.GroupsFromBP(customizations.GetGroups()) - img.KernelOptionsAppend = []string{ "rw", // TODO: Drop this as we expect kargs to come from the container image, @@ -88,8 +86,6 @@ func manifestForDiskImage(c *ManifestConfig, rng *rand.Rand) (*manifest.Manifest "console=ttyS0", } - img.SysrootReadOnly = true - switch c.Architecture { case arch.ARCH_X86_64: img.Platform = &platform.X86{ diff --git a/bib/cmd/bootc-image-builder/main_test.go b/bib/cmd/bootc-image-builder/main_test.go index 804479be..fb8815d2 100644 --- a/bib/cmd/bootc-image-builder/main_test.go +++ b/bib/cmd/bootc-image-builder/main_test.go @@ -67,7 +67,6 @@ func getBaseConfig() *main.ManifestConfig { func getUserConfig() *main.ManifestConfig { // add a user - pass := "super-secret-password-42" key := "ssh-ed25519 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" return &main.ManifestConfig{ Imgref: "testuser", @@ -77,9 +76,8 @@ func getUserConfig() *main.ManifestConfig { Customizations: &blueprint.Customizations{ User: []blueprint.UserCustomization{ { - Name: "tester", - Password: &pass, - Key: &key, + Name: "root", + Key: &key, }, }, }, @@ -173,7 +171,7 @@ func TestManifestSerialization(t *testing.T) { "build": { containerSpec, }, - "ostree-deployment": { + "image": { containerSpec, }, } @@ -230,15 +228,12 @@ func TestManifestSerialization(t *testing.T) { containers: diskContainers, expStages: map[string][]string{ "build": {"org.osbuild.container-deploy"}, - "ostree-deployment": { - "org.osbuild.ostree.deploy.container", + "image": { + "org.osbuild.bootc.install-to-filesystem", }, }, nexpStages: map[string][]string{ "build": {"org.osbuild.rpm"}, - "ostree-deployment": { - "org.osbuild.users", - }, }, }, "raw-base": { @@ -247,15 +242,12 @@ func TestManifestSerialization(t *testing.T) { containers: diskContainers, expStages: map[string][]string{ "build": {"org.osbuild.container-deploy"}, - "ostree-deployment": { - "org.osbuild.ostree.deploy.container", + "image": { + "org.osbuild.bootc.install-to-filesystem", }, }, nexpStages: map[string][]string{ "build": {"org.osbuild.rpm"}, - "ostree-deployment": { - "org.osbuild.users", - }, }, }, "qcow2-base": { @@ -264,15 +256,12 @@ func TestManifestSerialization(t *testing.T) { containers: diskContainers, expStages: map[string][]string{ "build": {"org.osbuild.container-deploy"}, - "ostree-deployment": { - "org.osbuild.ostree.deploy.container", + "image": { + "org.osbuild.bootc.install-to-filesystem", }, }, nexpStages: map[string][]string{ "build": {"org.osbuild.rpm"}, - "ostree-deployment": { - "org.osbuild.users", - }, }, }, "ami-user": { @@ -281,9 +270,8 @@ func TestManifestSerialization(t *testing.T) { containers: diskContainers, expStages: map[string][]string{ "build": {"org.osbuild.container-deploy"}, - "ostree-deployment": { - "org.osbuild.users", - "org.osbuild.ostree.deploy.container", + "image": { + "org.osbuild.bootc.install-to-filesystem", }, }, nexpStages: map[string][]string{ @@ -296,9 +284,8 @@ func TestManifestSerialization(t *testing.T) { containers: diskContainers, expStages: map[string][]string{ "build": {"org.osbuild.container-deploy"}, - "ostree-deployment": { - "org.osbuild.users", // user creation stage when we add users - "org.osbuild.ostree.deploy.container", + "image": { + "org.osbuild.bootc.install-to-filesystem", }, }, nexpStages: map[string][]string{ @@ -311,9 +298,8 @@ func TestManifestSerialization(t *testing.T) { containers: diskContainers, expStages: map[string][]string{ "build": {"org.osbuild.container-deploy"}, - "ostree-deployment": { - "org.osbuild.users", // user creation stage when we add users - "org.osbuild.ostree.deploy.container", + "image": { + "org.osbuild.bootc.install-to-filesystem", }, }, nexpStages: map[string][]string{ @@ -343,20 +329,23 @@ func TestManifestSerialization(t *testing.T) { packages: isoPackages, err: "missing ostree, container, or ospipeline parameters in ISO tree pipeline", }, + // the errors here when the containers are misisng are + // confusing, we should probably not test for them at + // this level "ami-nocontainer": { config: userConfig, imageTypes: []string{"ami"}, - err: "pipeline ostree-deployment requires exactly one ostree commit or one container (have commits: []; containers: [])", + err: "serialization not started", }, "raw-nocontainer": { config: userConfig, imageTypes: []string{"raw"}, - err: "pipeline ostree-deployment requires exactly one ostree commit or one container (have commits: []; containers: [])", + err: "serialization not started", }, "qcow2-nocontainer": { config: userConfig, imageTypes: []string{"qcow2"}, - err: "pipeline ostree-deployment requires exactly one ostree commit or one container (have commits: []; containers: [])", + err: "serialization not started", }, } From 9167b9ab7e57ee7740f48b74aa03cc6a85b9ad51 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 Mar 2024 09:53:56 +0100 Subject: [PATCH 2/8] tests: rename nexpStages to notExpectedStages On casual reading `nexp` reads almost like a typo of `next` and it's generally a bit short. Let's make it very explicit instead :) --- bib/cmd/bootc-image-builder/main_test.go | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bib/cmd/bootc-image-builder/main_test.go b/bib/cmd/bootc-image-builder/main_test.go index fb8815d2..80fbb398 100644 --- a/bib/cmd/bootc-image-builder/main_test.go +++ b/bib/cmd/bootc-image-builder/main_test.go @@ -52,13 +52,13 @@ func TestCanChownInPathCannotChange(t *testing.T) { } type manifestTestCase struct { - config *main.ManifestConfig - imageTypes []string - packages map[string][]rpmmd.PackageSpec - containers map[string][]container.Spec - expStages map[string][]string - nexpStages map[string][]string - err interface{} + config *main.ManifestConfig + imageTypes []string + packages map[string][]rpmmd.PackageSpec + containers map[string][]container.Spec + expStages map[string][]string + notExpectedStages map[string][]string + err interface{} } func getBaseConfig() *main.ManifestConfig { @@ -232,7 +232,7 @@ func TestManifestSerialization(t *testing.T) { "org.osbuild.bootc.install-to-filesystem", }, }, - nexpStages: map[string][]string{ + notExpectedStages: map[string][]string{ "build": {"org.osbuild.rpm"}, }, }, @@ -246,7 +246,7 @@ func TestManifestSerialization(t *testing.T) { "org.osbuild.bootc.install-to-filesystem", }, }, - nexpStages: map[string][]string{ + notExpectedStages: map[string][]string{ "build": {"org.osbuild.rpm"}, }, }, @@ -260,7 +260,7 @@ func TestManifestSerialization(t *testing.T) { "org.osbuild.bootc.install-to-filesystem", }, }, - nexpStages: map[string][]string{ + notExpectedStages: map[string][]string{ "build": {"org.osbuild.rpm"}, }, }, @@ -274,7 +274,7 @@ func TestManifestSerialization(t *testing.T) { "org.osbuild.bootc.install-to-filesystem", }, }, - nexpStages: map[string][]string{ + notExpectedStages: map[string][]string{ "build": {"org.osbuild.rpm"}, }, }, @@ -288,7 +288,7 @@ func TestManifestSerialization(t *testing.T) { "org.osbuild.bootc.install-to-filesystem", }, }, - nexpStages: map[string][]string{ + notExpectedStages: map[string][]string{ "build": {"org.osbuild.rpm"}, }, }, @@ -302,7 +302,7 @@ func TestManifestSerialization(t *testing.T) { "org.osbuild.bootc.install-to-filesystem", }, }, - nexpStages: map[string][]string{ + notExpectedStages: map[string][]string{ "build": {"org.osbuild.rpm"}, }, }, @@ -368,7 +368,7 @@ func TestManifestSerialization(t *testing.T) { } else { manifestJson, err := mf.Serialize(tc.packages, tc.containers, nil) assert.NoError(err) - assert.NoError(checkStages(manifestJson, tc.expStages, tc.nexpStages)) + assert.NoError(checkStages(manifestJson, tc.expStages, tc.notExpectedStages)) } }) } From 84fcd500d212ae117080916f6d1dc62becc695fe Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 Mar 2024 11:35:06 +0100 Subject: [PATCH 3/8] bib: add -experimental-root-ssh-authorized-key option With the move to bootc install to-filesystem we cannot currently customize users except for roots ssh authorized_keys. So make this the only allowed option. Note that it would be nicer if that would be symetric to bootc --root-ssh-authorized-keys that takes a file but our image customizations currently only support a single key. So start here by marking it experimental to unblock tests and fix images to eventually be symetric. --- bib/cmd/bootc-image-builder/main.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/bib/cmd/bootc-image-builder/main.go b/bib/cmd/bootc-image-builder/main.go index d4c55166..b72125bc 100644 --- a/bib/cmd/bootc-image-builder/main.go +++ b/bib/cmd/bootc-image-builder/main.go @@ -11,7 +11,10 @@ import ( "strconv" "strings" - "github.com/osbuild/bootc-image-builder/bib/internal/setup" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "golang.org/x/exp/slices" + "github.com/osbuild/images/pkg/arch" "github.com/osbuild/images/pkg/blueprint" "github.com/osbuild/images/pkg/cloud/awscloud" @@ -20,9 +23,8 @@ import ( "github.com/osbuild/images/pkg/manifest" "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/rpmmd" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "golang.org/x/exp/slices" + + "github.com/osbuild/bootc-image-builder/bib/internal/setup" ) //go:embed fedora-eln.json @@ -239,6 +241,7 @@ func manifestFromCobra(cmd *cobra.Command, args []string) ([]byte, error) { targetArch, _ := cmd.Flags().GetString("target-arch") tlsVerify, _ := cmd.Flags().GetBool("tls-verify") localStorage, _ := cmd.Flags().GetBool("local") + rootSSHKey, _ := cmd.Flags().GetString("experimental-root-ssh-authorized-key") if targetArch != "" { // TODO: detect if binfmt_misc for target arch is @@ -269,6 +272,16 @@ func manifestFromCobra(cmd *cobra.Command, args []string) ([]byte, error) { config = &BuildConfig{} } + if rootSSHKey != "" { + config.Blueprint = &blueprint.Blueprint{ + Customizations: &blueprint.Customizations{ + User: []blueprint.UserCustomization{ + {Name: "root", Key: &rootSSHKey}, + }, + }, + } + } + manifestConfig := &ManifestConfig{ Architecture: buildArch, Config: config, @@ -460,6 +473,8 @@ func run() error { manifestCmd.Flags().String("target-arch", "", "build for the given target architecture (experimental)") manifestCmd.Flags().StringArray("type", []string{"qcow2"}, fmt.Sprintf("image types to build [%s]", allImageTypesString())) manifestCmd.Flags().Bool("local", false, "use a local container rather than a container from a registry") + // XXX: hide from help? + manifestCmd.Flags().String("experimental-root-ssh-authorized-key", "", "authorized ssh key for root as string") logrus.SetLevel(logrus.ErrorLevel) buildCmd.Flags().AddFlagSet(manifestCmd.Flags()) From 1455c74fb8a467490f532f1b3d88e6e829bc1579 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 Mar 2024 12:01:13 +0100 Subject: [PATCH 4/8] README.md: remove all traces of -config, update example This commit removes the -config options from bib. Blueprint customizations are no longer supported with the move to bootc install to-filesystem. This is fine but we need to reflect it in our docs. It also updates the example with an example how to create an image with an user. This might be slightly controversial but I think it's rather important to give users a quick path to success (and the updated README also links to the upstream docs for a more indepth discussion). --- README.md | 102 ++++++++---------------------------------------------- 1 file changed, 14 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 9340036b..419e1fd9 100644 --- a/README.md +++ b/README.md @@ -23,32 +23,35 @@ $ podman machine start ## 🚀 Examples -The following example builds a [Fedora ELN](https://docs.fedoraproject.org/en-US/eln/) bootable container into a QCOW2 image for the architecture you're running -the command on. - -The `fedora-bootc:eln` base image does not include a default user. This example injects a [user configuration file](#-build-config) -by adding a volume-mount for the local file as well as the `--config` flag to the bootc-image-builder container. - -The following command will create a QCOW2 disk image. First, create `./config.json` as described above to configure user access. +The following example builds a derrived [Fedora ELN](https://docs.fedoraproject.org/en-US/eln/) bootable container into a QCOW2 image for the architecture you're running the command on. ```bash +cat > Containerfile <<'EOF' +FROM quay.io/centos-bootc/fedora-bootc:eln +RUN useradd alice --password "$(openssl passwd -6 "bob")" --groups wheel +EOF +sudo podman build -f Containerfile -t my-bootc sudo podman run \ --rm \ -it \ --privileged \ --pull=newer \ --security-opt label=type:unconfined_t \ - -v $(pwd)/config.json:/config.json \ -v $(pwd)/output:/output \ + -v /var/lib/containers/storage:/var/lib/containers/storage \ quay.io/centos-bootc/bootc-image-builder:latest \ --type qcow2 \ - --config /config.json \ - quay.io/centos-bootc/fedora-bootc:eln + --local \ + localhost/my-bootc ``` +There is more in-depth discussion how to customize the image and the users in the [bootc documentation](https://bootc-org.gitlab.io/documentation/guide/building.html#logins-and-users). + ### Using local containers -To use containers from local container's storage rather than a registry, we need to ensure two things: +By default (without the `--local` flag) bootc-image-builder will use +the podman registry. To use containers from local container's storage, +we need to ensure two things: - the container exists in local storage - mount the local container storage @@ -61,12 +64,10 @@ sudo podman run \ --privileged \ --pull=newer \ --security-opt label=type:unconfined_t \ - -v $(pwd)/config.json:/config.json \ -v $(pwd)/output:/output \ -v /var/lib/containers/storage:/var/lib/containers/storage \ quay.io/centos-bootc/bootc-image-builder:latest \ --type qcow2 \ - --config /config.json \ --local \ localhost/bootc:eln ``` @@ -134,7 +135,6 @@ Usage: Flags: --chown string chown the ouput directory to match the specified UID:GID - --config string build config file --tls-verify require HTTPS and verify certificates when contacting registries (default true) --type string image type to build [qcow2, ami] (default "qcow2") ``` @@ -144,7 +144,6 @@ Flags: | Argument | Description | Default Value | |------------------|------------------------------------------------------------------|:-------------:| | **--chown** | chown the ouput directory to match the specified UID:GID | ❌ | -| **--config** | Path to a [build config](#-build-config) | ❌ | | **--tls-verify** | Require HTTPS and verify certificates when contacting registries | `true` | | **--type** | [Image type](#-image-types) to build | `qcow2` | @@ -262,79 +261,6 @@ The following volumes can be mounted inside the container: | `/store` | Used for the [osbuild store](https://www.osbuild.org/) | No | | `/rpmmd` | Used for the DNF cache | No | -## 📝 Build config - -A build config is a JSON file with customizations for the resulting image. A path to the file is passed via the `--config` argument. The customizations are specified under a `blueprint.customizations` object. - -As an example, let's show how you can add a user to the image: - -Firstly create a file `./config.json` and put the following content into it: - -```json -{ - "blueprint": { - "customizations": { - "user": [ - { - "name": "alice", - "password": "bob", - "key": "ssh-rsa AAA ... user@email.com", - "groups": [ - "wheel" - ] - } - ] - } - } -} -``` - -Then, run `bootc-image-builder` with the following arguments: - -```bash -sudo podman run \ - --rm \ - -it \ - --privileged \ - --pull=newer \ - --security-opt label=type:unconfined_t \ - -v $(pwd)/config.json:/config.json \ - -v $(pwd)/output:/output \ - quay.io/centos-bootc/bootc-image-builder:latest \ - --type qcow2 \ - --config /config.json \ - quay.io/centos-bootc/fedora-bootc:eln -``` - -### Users (`user`, array) - -Possible fields: - -| Field | Use | Required | -|------------|--------------------------------------------|:--------:| -| `name` | Name of the user | ✅ | -| `password` | Unencrypted password | No | -| `key` | Public SSH key contents | No | -| `groups` | An array of secondary to put the user into | No | - -Example: - -```json -{ - "user": [ - { - "name": "alice", - "password": "bob", - "key": "ssh-rsa AAA ... user@email.com", - "groups": [ - "wheel", - "admins" - ] - } - ] -} -``` - ## Building To build the container locally you can run From 2585c739dec7b2e2f712ff46f886555f44e5ccb7 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Mar 2024 17:57:45 +0100 Subject: [PATCH 5/8] go.{mod,sum}: move to latest osbuild/images for bootc install to-filesystem support --- bib/go.mod | 2 +- bib/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bib/go.mod b/bib/go.mod index c54c95d6..01efd284 100644 --- a/bib/go.mod +++ b/bib/go.mod @@ -7,7 +7,7 @@ require ( github.com/cheggaaa/pb/v3 v3.1.5 github.com/google/uuid v1.6.0 github.com/moby/sys/mountinfo v0.7.1 - github.com/osbuild/images v0.50.0 + github.com/osbuild/images v0.51.1-0.20240326154530-58ef1aea0410 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/bib/go.sum b/bib/go.sum index efe228d8..39dda266 100644 --- a/bib/go.sum +++ b/bib/go.sum @@ -279,6 +279,8 @@ github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaL github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/osbuild/images v0.50.0 h1:UONHoTONdgOLBZ6ZNXKLsnMefgq3bccgxBvmlRyEhbk= github.com/osbuild/images v0.50.0/go.mod h1:eM/J8+hEUH0jrwcy3DtE6SDg+bRMWFZIf5d+YDyhoDY= +github.com/osbuild/images v0.51.1-0.20240326154530-58ef1aea0410 h1:93/m6O0nH1sU4TY2Dlh6lFfaS8gHQP8hAb3HnTrCS3E= +github.com/osbuild/images v0.51.1-0.20240326154530-58ef1aea0410/go.mod h1:eM/J8+hEUH0jrwcy3DtE6SDg+bRMWFZIf5d+YDyhoDY= github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f h1:/UDgs8FGMqwnHagNDPGOlts35QkhAZ8by3DR7nMih7M= github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= From 0fb38da50eb2ce33866183e246b3259f54fd5945 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Mar 2024 18:00:33 +0100 Subject: [PATCH 6/8] bib: remove img.ISOLabelTmpl This was removed in https://github.com/osbuild/images/pull/542 --- bib/cmd/bootc-image-builder/image.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/bib/cmd/bootc-image-builder/image.go b/bib/cmd/bootc-image-builder/image.go index 57c8fc81..e590c9b6 100644 --- a/bib/cmd/bootc-image-builder/image.go +++ b/bib/cmd/bootc-image-builder/image.go @@ -261,8 +261,6 @@ func manifestForISO(c *ManifestConfig, rng *rand.Rand) (*manifest.Manifest, erro }, } - img.ISOLabelTmpl = "Container-Installer-%s" - var customizations *blueprint.Customizations if c.Config != nil && c.Config.Blueprint != nil { customizations = c.Config.Blueprint.Customizations From 811af24e484548c56c686251030d929e901e09c5 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 26 Mar 2024 18:56:24 +0100 Subject: [PATCH 7/8] bib: rename `--config` to `--iso-config` Rename the --config option to --iso-config and error if it is used for anything other than the ISO image. --- bib/cmd/bootc-image-builder/main.go | 19 ++++++++------ test/test_opts.py | 39 +++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/bib/cmd/bootc-image-builder/main.go b/bib/cmd/bootc-image-builder/main.go index b72125bc..c57d639d 100644 --- a/bib/cmd/bootc-image-builder/main.go +++ b/bib/cmd/bootc-image-builder/main.go @@ -235,7 +235,7 @@ func manifestFromCobra(cmd *cobra.Command, args []string) ([]byte, error) { } imgref := args[0] - configFile, _ := cmd.Flags().GetString("config") + isoConfigFile, _ := cmd.Flags().GetString("iso-config") imgTypes, _ := cmd.Flags().GetStringArray("type") rpmCacheRoot, _ := cmd.Flags().GetString("rpmmd") targetArch, _ := cmd.Flags().GetString("target-arch") @@ -262,11 +262,16 @@ func manifestFromCobra(cmd *cobra.Command, args []string) ([]byte, error) { return nil, err } + // bootc does not yet support arbitray blueprint customizations + if buildType != BuildTypeISO && isoConfigFile != "" { + return nil, fmt.Errorf("the --iso-config switch is only supported for ISO images") + } + var config *BuildConfig - if configFile != "" { - config, err = loadConfig(configFile) + if isoConfigFile != "" { + config, err = loadConfig(isoConfigFile) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot load config: %w", err) } } else { config = &BuildConfig{} @@ -351,7 +356,7 @@ func cmdBuild(cmd *cobra.Command, args []string) error { fmt.Printf("Generating manifest %s\n", manifest_fname) mf, err := manifestFromCobra(cmd, args) if err != nil { - panic(err) + return err } fmt.Print("DONE\n") @@ -468,7 +473,7 @@ func run() error { } rootCmd.AddCommand(manifestCmd) manifestCmd.Flags().Bool("tls-verify", true, "require HTTPS and verify certificates when contacting registries") - manifestCmd.Flags().String("config", "", "build config file") + manifestCmd.Flags().String("iso-config", "", "build config file for the iso") manifestCmd.Flags().String("rpmmd", "/rpmmd", "rpm metadata cache directory") manifestCmd.Flags().String("target-arch", "", "build for the given target architecture (experimental)") manifestCmd.Flags().StringArray("type", []string{"qcow2"}, fmt.Sprintf("image types to build [%s]", allImageTypesString())) @@ -492,7 +497,7 @@ func run() error { return err } } - if err := buildCmd.MarkFlagFilename("config"); err != nil { + if err := buildCmd.MarkFlagFilename("iso-config"); err != nil { return err } buildCmd.MarkFlagsRequiredTogether("aws-region", "aws-bucket", "aws-ami-name") diff --git a/test/test_opts.py b/test/test_opts.py index 5e3159a3..8c1b224a 100644 --- a/test/test_opts.py +++ b/test/test_opts.py @@ -1,4 +1,5 @@ import subprocess +import sys import pytest @@ -27,3 +28,41 @@ def test_bib_chown_opts(tmp_path, build_fake_container, chown_opt, expected_uid_ assert p.exists() assert p.stat().st_uid == expected_uid_gid[0] assert p.stat().st_gid == expected_uid_gid[1] + + +def test_bib_config_errors_for_default(tmp_path, build_fake_container): + output_path = tmp_path / "output" + output_path.mkdir(exist_ok=True) + + ret = subprocess.run([ + "podman", "run", "--rm", + "--privileged", + "--security-opt", "label=type:unconfined_t", + "-v", f"{output_path}:/output", + build_fake_container, + "--iso-config", "/some/random/config", + "quay.io/centos-bootc/centos-bootc:stream9", + ], check=False, encoding="utf8", stdout=sys.stdout, stderr=subprocess.PIPE) + assert ret.returncode != 0 + assert "the --iso-config switch is only supported for ISO images" in ret.stderr + + +def test_bib_iso_config_is_parsed(tmp_path, build_fake_container): + output_path = tmp_path / "output" + output_path.mkdir(exist_ok=True) + + # check that config.json is tried to be loaded + (tmp_path / "config.json").write_text("invalid-json", encoding="utf8") + ret = subprocess.run([ + "podman", "run", "--rm", + "--privileged", + "--security-opt", "label=type:unconfined_t", + "-v", f"{output_path}:/output", + "-v", f"{tmp_path}/config.json:/config.json", + build_fake_container, + "--iso-config", "/config.json", + "--type", "anaconda-iso", + "quay.io/centos-bootc/centos-bootc:stream9", + ], check=False, encoding="utf8", stdout=sys.stdout, stderr=subprocess.PIPE) + assert ret.returncode != 0 + assert "cannot load config: invalid character" in ret.stderr From 1ebdf76eeebf9b5be4ff5992272c8098df8c42d0 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 22 Mar 2024 12:37:56 +0100 Subject: [PATCH 8/8] test: update tests to use ssh keys instead of passwords With the switch to bootc we need to adjust the testing. We inject a root ssh key now and just use that for login. --- test/test_build.py | 62 +++++++++++++++++++++++----------------------- test/vm.py | 8 ++++-- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/test/test_build.py b/test/test_build.py index 174e69fc..ea755e66 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -1,4 +1,3 @@ -import json import os import pathlib import platform @@ -37,7 +36,7 @@ class ImageBuildResult(NamedTuple): img_path: str img_arch: str username: str - password: str + ssh_keyfile_private_path: str bib_output: str journal_output: str metadata: dict = {} @@ -123,9 +122,6 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): # split and check each one image_types = images.split("+") - username = "test" - password = "password" - # params can be long and the qmp socket (that has a limit of 100ish # AF_UNIX) is derived from the path # hash the container_ref+target_arch, but exclude the image_type so that the output path is shared between calls to @@ -135,6 +131,10 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): journal_log_path = output_path / "journal.log" bib_output_path = output_path / "bib-output.log" + + ssh_keyfile_private_path = output_path / "ssh-keyfile" + ssh_keyfile_public_path = ssh_keyfile_private_path.with_suffix(".pub") + artifact = { "qcow2": pathlib.Path(output_path) / "qcow2/disk.qcow2", "ami": pathlib.Path(output_path) / "image/disk.raw", @@ -166,9 +166,22 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): journal_output = journal_log_path.read_text(encoding="utf8") bib_output = bib_output_path.read_text(encoding="utf8") results.append(ImageBuildResult( - image_type, generated_img, target_arch, username, password, + image_type, generated_img, target_arch, + "root", ssh_keyfile_private_path, bib_output, journal_output)) + # generate new keyfile + if not ssh_keyfile_private_path.exists(): + subprocess.run([ + "ssh-keygen", + "-N", "", + # be very conservative with keys for paramiko + "-b", "2048", + "-t", "rsa", + "-f", os.fspath(ssh_keyfile_private_path), + ], check=True) + ssh_pubkey = ssh_keyfile_public_path.read_text(encoding="utf8") + # Because we always build all image types, regardless of what was requested, we should either have 0 results or all # should be available, so if we found at least one result but not all of them, this is a problem with our setup assert not results or len(results) == len(image_types), \ @@ -181,23 +194,6 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): print(f"Requested {len(image_types)} images but found {len(results)} cached images. Building...") # not all requested image types are available - build them - cfg = { - "blueprint": { - "customizations": { - "user": [ - { - "name": username, - "password": password, - "groups": ["wheel"], - }, - ], - }, - }, - } - - config_json_path = output_path / "config.json" - config_json_path.write_text(json.dumps(cfg), encoding="utf-8") - cursor = testutil.journal_cursor() upload_args = [] @@ -245,7 +241,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload): *creds_args, build_container, container_ref, - "--config", "/output/config.json", + "--experimental-root-ssh-authorized-key", ssh_pubkey, *types_arg, *upload_args, *target_arch_args, @@ -281,8 +277,10 @@ def del_ami(): results = [] for image_type in image_types: - results.append(ImageBuildResult(image_type, artifact[image_type], target_arch, - username, password, bib_output, journal_output, metadata)) + results.append(ImageBuildResult( + image_type, artifact[image_type], target_arch, + "root", ssh_keyfile_private_path, + bib_output, journal_output, metadata)) yield results # Try to cache as much as possible @@ -318,9 +316,10 @@ def test_image_is_generated(image_type): @pytest.mark.parametrize("image_type", gen_testcases("qemu-boot"), indirect=["image_type"]) def test_image_boots(image_type): with QEMU(image_type.img_path, arch=image_type.img_arch) as test_vm: - exit_status, _ = test_vm.run("true", user=image_type.username, password=image_type.password) + exit_status, _ = test_vm.run("true", user=image_type.username, keyfile=image_type.ssh_keyfile_private_path) assert exit_status == 0 - exit_status, output = test_vm.run("echo hello", user=image_type.username, password=image_type.password) + exit_status, output = test_vm.run( + "echo hello", user=image_type.username, keyfile=image_type.ssh_keyfile_private_path) assert exit_status == 0 assert "hello" in output @@ -337,9 +336,10 @@ def test_ami_boots_in_aws(image_type, force_aws_upload): # 4.30 GiB / 10.00 GiB [------------>____________] 43.02% 58.04 MiB p/s assert "] 100.00%" in image_type.bib_output with AWS(image_type.metadata["ami_id"]) as test_vm: - exit_status, _ = test_vm.run("true", user=image_type.username, password=image_type.password) + exit_status, _ = test_vm.run("true", user=image_type.username, keyfile=image_type.ssh_keyfile_private_path) assert exit_status == 0 - exit_status, output = test_vm.run("echo hello", user=image_type.username, password=image_type.password) + exit_status, output = test_vm.run( + "echo hello", user=image_type.username, keyfile=image_type.ssh_keyfile_private_path) assert exit_status == 0 assert "hello" in output @@ -401,7 +401,7 @@ def test_iso_installs(image_type): # boot test disk and do extremly simple check with QEMU(test_disk_path) as vm: vm.start(use_ovmf=True) - exit_status, _ = vm.run("true", user=image_type.username, password=image_type.password) + exit_status, _ = vm.run("true", user=image_type.username, keyfile=image_type.ssh_keyfile_private_path) assert exit_status == 0 diff --git a/test/vm.py b/test/vm.py index 43c37636..b808c078 100644 --- a/test/vm.py +++ b/test/vm.py @@ -1,5 +1,6 @@ import abc import os +import paramiko import pathlib import platform import subprocess @@ -43,7 +44,7 @@ def force_stop(self): Stop the VM and clean up any resources that were created when setting up and starting the machine. """ - def run(self, cmd, user, password): + def run(self, cmd, user, keyfile): """ Run a command on the VM via SSH using the provided credentials. """ @@ -51,8 +52,11 @@ def run(self, cmd, user, password): self.start() client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy) + # workaround, see https://github.com/paramiko/paramiko/issues/2048 + pkey = paramiko.RSAKey.from_private_key_file(keyfile) client.connect( - self._address, self._ssh_port, user, password, + self._address, self._ssh_port, + user, pkey=pkey, allow_agent=False, look_for_keys=False) chan = client.get_transport().open_session() chan.get_pty()