Skip to content

Commit

Permalink
Implementation of the multi-platform RFC - 0128
Browse files Browse the repository at this point in the history
Signed-off-by: Juan Bustamante <[email protected]>
  • Loading branch information
jjbustamante committed May 8, 2024
1 parent 67feb16 commit cb4b368
Show file tree
Hide file tree
Showing 28 changed files with 1,006 additions and 150 deletions.
79 changes: 56 additions & 23 deletions acceptance/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,34 +255,67 @@ func testWithoutSpecificBuilderRequirement(
})

when("--publish", func() {
it("publishes image to registry", func() {
packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10))
it.Before(func() {
// used to avoid authentication issues with the local registry
os.Setenv("DOCKER_CONFIG", registryConfig.DockerConfigDir)
})

nestedPackage := buildpacks.NewPackageImage(
t,
pack,
nestedPackageName,
packageTomlPath,
buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
buildpacks.WithPublish(),
)
buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)
when("no --targets", func() {
it("publishes image to registry", func() {
packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
nestedPackageName := registryConfig.RepoName("test/package-" + h.RandString(10))

aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
packageName := registryConfig.RepoName("test/package-" + h.RandString(10))
nestedPackage := buildpacks.NewPackageImage(
t,
pack,
nestedPackageName,
packageTomlPath,
buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers),
buildpacks.WithPublish(),
)
buildpackManager.PrepareBuildModules(tmpDir, nestedPackage)

output := pack.RunSuccessfully(
"buildpack", "package", packageName,
"-c", aggregatePackageToml,
"--publish",
)
aggregatePackageToml := generateAggregatePackageToml("simple-layers-parent-buildpack.tgz", nestedPackageName, imageManager.HostOS())
packageName := registryConfig.RepoName("test/package-" + h.RandString(10))

defer imageManager.CleanupImages(packageName)
assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)
output := pack.RunSuccessfully(
"buildpack", "package", packageName,
"-c", aggregatePackageToml,
"--publish",
)

defer imageManager.CleanupImages(packageName)
assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)

assertImage.NotExistsLocally(packageName)
assertImage.CanBePulledFromRegistry(packageName)
assertImage.NotExistsLocally(packageName)
assertImage.CanBePulledFromRegistry(packageName)
})
})

when("--targets", func() {
it("publishes images to registry and creates an image index", func() {
h.SkipIf(t, !pack.SupportsFeature(invoke.MultiPlatformBuildersAndBuildPackages), "multi-platform builders and buildpack packages are available since 0.34.0")

packageName := registryConfig.RepoName("simple-multi-platform-buildpack" + h.RandString(8))
packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, "package.toml", imageManager.HostOS())

output := pack.RunSuccessfully(
"buildpack", "package", packageName,
"-c", packageTomlPath,
"--publish",
"--target", "linux/amd64",
"--target", "windows/amd64",
)

defer imageManager.CleanupImages(packageName)
assertions.NewOutputAssertionManager(t, output).ReportsPackagePublished(packageName)

assertImage.NotExistsLocally(packageName)
assertImage.CanBePulledFromRegistry(packageName)

assertions.NewOutputAssertionManager(t, output).ReportsSuccessfulIndexPushed(packageName)
h.AssertRemoteImageIndex(t, packageName, types.OCIImageIndex, 2)
})
})
})

Expand Down
7 changes: 7 additions & 0 deletions acceptance/buildpacks/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func (b BuildModuleManager) PrepareBuildModules(destination string, modules ...T
type Modifiable interface {
SetPublish()
SetBuildpacks([]TestBuildModule)
SetTargets([]string)
}
type PackageModifier func(p Modifiable)

Expand All @@ -62,3 +63,9 @@ func WithPublish() PackageModifier {
p.SetPublish()
}
}

func WithTargets(targets []string) PackageModifier {
return func(p Modifiable) {
p.SetTargets(targets)
}
}
2 changes: 2 additions & 0 deletions acceptance/buildpacks/package_file_buildpack.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ func (p *PackageFile) SetBuildpacks(buildpacks []TestBuildModule) {

func (p *PackageFile) SetPublish() {}

func (p *PackageFile) SetTargets(_ []string) {}

func NewPackageFile(
t *testing.T,
pack *invoke.PackInvoker,
Expand Down
11 changes: 11 additions & 0 deletions acceptance/buildpacks/package_image_buildpack.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type PackageImage struct {
sourceConfigLocation string
buildpacks []TestBuildModule
publish bool
targets []string
}

func (p *PackageImage) SetBuildpacks(buildpacks []TestBuildModule) {
Expand All @@ -33,6 +34,10 @@ func (p *PackageImage) SetPublish() {
p.publish = true
}

func (p *PackageImage) SetTargets(targets []string) {
p.targets = targets
}

func NewPackageImage(
t *testing.T,
pack *invoke.PackInvoker,
Expand Down Expand Up @@ -81,6 +86,12 @@ func (p PackageImage) Prepare(sourceDir, _ string) error {

if p.publish {
packArgs = append(packArgs, "--publish")
if len(p.targets) > 0 {
for _, t := range p.targets {
packArgs = append(packArgs, "--target", t)
}
fmt.Println(packArgs)
}
}

p.testObject.Log("packaging image: ", p.name)
Expand Down
4 changes: 4 additions & 0 deletions acceptance/invoke/pack.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ const (
FlattenBuilderCreationV2
FixesRunImageMetadata
ManifestCommands
MultiPlatformBuildersAndBuildPackages
)

var featureTests = map[Feature]func(i *PackInvoker) bool{
Expand Down Expand Up @@ -278,6 +279,9 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
ManifestCommands: func(i *PackInvoker) bool {
return i.atLeast("v0.34.0")
},
MultiPlatformBuildersAndBuildPackages: func(i *PackInvoker) bool {
return i.atLeast("v0.34.0")
},
}

func (i *PackInvoker) SupportsFeature(f Feature) bool {
Expand Down
1 change: 1 addition & 0 deletions builder/config_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type Config struct {
Lifecycle LifecycleConfig `toml:"lifecycle"`
Run RunConfig `toml:"run"`
Build BuildConfig `toml:"build"`
Targets []dist.Target `toml:"targets"`
}

// ModuleCollection is a list of ModuleConfigs
Expand Down
17 changes: 16 additions & 1 deletion buildpackage/config_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@ type Config struct {
Buildpack dist.BuildpackURI `toml:"buildpack"`
Extension dist.BuildpackURI `toml:"extension"`
Dependencies []dist.ImageOrURI `toml:"dependencies"`
Platform dist.Platform `toml:"platform"`
// deprecated
Platform dist.Platform `toml:"platform"`

// Define targets for composite buildpacks
Targets []dist.Target `toml:"targets"`
}

func DefaultConfig() Config {
Expand Down Expand Up @@ -117,6 +121,17 @@ func (r *ConfigReader) Read(path string) (Config, error) {
return packageConfig, nil
}

func (r *ConfigReader) ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error) {
buildpackCfg := dist.BuildpackDescriptor{}

_, err := toml.DecodeFile(path, &buildpackCfg)
if err != nil {
return dist.BuildpackDescriptor{}, err
}

return buildpackCfg, nil
}

func validateURI(uri, relativeBaseDir string) error {
locatorType, err := buildpack.GetLocatorType(uri, relativeBaseDir, nil)
if err != nil {
Expand Down
17 changes: 17 additions & 0 deletions internal/commands/builder_create.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ type BuilderCreateFlags struct {
Registry string
Policy string
Flatten []string
Targets []string
Label map[string]string
}

Expand Down Expand Up @@ -87,6 +88,15 @@ Creating a custom builder allows you to control what buildpacks are used and wha
return err
}

multiArchCfg, err := processMultiArchitectureConfig(logger, flags.Targets, builderConfig.Targets, !flags.Publish)
if err != nil {
return err
}

if len(multiArchCfg.Targets()) == 0 {
logger.Warnf("A new '--target' flag is available to set the platform")
}

imageName := args[0]
if err := pack.CreateBuilder(cmd.Context(), client.CreateBuilderOptions{
RelativeBaseDir: relativeBaseDir,
Expand All @@ -98,6 +108,7 @@ Creating a custom builder allows you to control what buildpacks are used and wha
PullPolicy: pullPolicy,
Flatten: toFlatten,
Labels: flags.Label,
Targets: multiArchCfg.Targets(),
}); err != nil {
return err
}
Expand All @@ -116,6 +127,12 @@ Creating a custom builder allows you to control what buildpacks are used and wha
cmd.Flags().StringVar(&flags.Policy, "pull-policy", "", "Pull policy to use. Accepted values are always, never, and if-not-present. The default is always")
cmd.Flags().StringArrayVar(&flags.Flatten, "flatten", nil, "List of buildpacks to flatten together into a single layer (format: '<buildpack-id>@<buildpack-version>,<buildpack-id>@<buildpack-version>'")
cmd.Flags().StringToStringVarP(&flags.Label, "label", "l", nil, "Labels to add to the builder image, in the form of '<name>=<value>'")
cmd.Flags().StringSliceVarP(&flags.Targets, "target", "t", nil,
`Target platforms to build for.\nTargets should be in the format '[os][/arch][/variant]:[distroname@osversion@anotherversion];[distroname@osversion]'.
- To specify two different architectures : '--target "linux/amd64" --target "linux/arm64"'
- To specify the distribution version: '--target "linux/arm/v6@[email protected]"'
- To specify multiple distribution versions : '--target "linux/arm/v6:[email protected]" --target "linux/arm/v6:[email protected]"'
`)

AddHelpFlag(cmd, "create")
return cmd
Expand Down
70 changes: 70 additions & 0 deletions internal/commands/buildpack_package.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package commands

import (
"context"
"os"
"path/filepath"
"strings"

Expand All @@ -12,6 +13,7 @@ import (
"github.com/buildpacks/pack/internal/config"
"github.com/buildpacks/pack/internal/style"
"github.com/buildpacks/pack/pkg/client"
"github.com/buildpacks/pack/pkg/dist"
"github.com/buildpacks/pack/pkg/image"
"github.com/buildpacks/pack/pkg/logging"
)
Expand All @@ -24,6 +26,7 @@ type BuildpackPackageFlags struct {
BuildpackRegistry string
Path string
FlattenExclude []string
Targets []string
Label map[string]string
Publish bool
Flatten bool
Expand All @@ -37,6 +40,7 @@ type BuildpackPackager interface {
// PackageConfigReader reads BuildpackPackage configs
type PackageConfigReader interface {
Read(path string) (pubbldpkg.Config, error)
ReadBuildpackDescriptor(path string) (dist.BuildpackDescriptor, error)
}

// BuildpackPackage packages (a) buildpack(s) into OCI format, based on a package config
Expand Down Expand Up @@ -100,6 +104,26 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa
logger.Warn("Flattening a buildpack package could break the distribution specification. Please use it with caution.")
}

targets, composedBP, err := processBuildpackPackageTargets(flags.Path, packageConfigReader, bpPackageCfg)
if err != nil {
return err
}
daemon := !flags.Publish && flags.Format == ""
multiArchCfg, err := processMultiArchitectureConfig(logger, flags.Targets, targets, daemon)
if err != nil {
return err
}

if len(multiArchCfg.Targets()) == 0 {
logger.Warnf("A new '--target' flag is available to set the platform, using '%s' as default", bpPackageCfg.Platform.OS)
} else if !composedBP {
filesToClean, err := multiArchCfg.CopyConfigFiles(relativeBaseDir)
if err != nil {
return err
}
defer clean(filesToClean)
}

if err := packager.PackageBuildpack(cmd.Context(), client.PackageBuildpackOptions{
RelativeBaseDir: relativeBaseDir,
Name: name,
Expand All @@ -111,6 +135,7 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa
Flatten: flags.Flatten,
FlattenExclude: flags.FlattenExclude,
Labels: flags.Label,
Targets: multiArchCfg.Targets(),
}); err != nil {
return err
}
Expand Down Expand Up @@ -138,6 +163,13 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa
cmd.Flags().BoolVar(&flags.Flatten, "flatten", false, "Flatten the buildpack into a single layer")
cmd.Flags().StringSliceVarP(&flags.FlattenExclude, "flatten-exclude", "e", nil, "Buildpacks to exclude from flattening, in the form of '<buildpack-id>@<buildpack-version>'")
cmd.Flags().StringToStringVarP(&flags.Label, "label", "l", nil, "Labels to add to packaged Buildpack, in the form of '<name>=<value>'")
cmd.Flags().StringSliceVarP(&flags.Targets, "target", "t", nil,
`Target platforms to build for.
Targets should be in the format '[os][/arch][/variant]:[distroname@osversion@anotherversion];[distroname@osversion]'.
- To specify two different architectures : '--target "linux/amd64" --target "linux/arm64"'
- To specify the distribution version: '--target "linux/arm/v6@[email protected]"'
- To specify multiple distribution versions : '--target "linux/arm/v6:[email protected]" --target "linux/arm/v6:[email protected]"'
`)
if !cfg.Experimental {
cmd.Flags().MarkHidden("flatten")
cmd.Flags().MarkHidden("flatten-exclude")
Expand Down Expand Up @@ -169,3 +201,41 @@ func validateBuildpackPackageFlags(cfg config.Config, p *BuildpackPackageFlags)
}
return nil
}

// processBuildpackPackageTargets returns the list of targets defined in the configuration file, it could be the buildpack.toml or
// the package.toml if the buildpack is a composed buildpack
func processBuildpackPackageTargets(path string, packageConfigReader PackageConfigReader, bpPackageCfg pubbldpkg.Config) ([]dist.Target, bool, error) {
var (
targets []dist.Target
order dist.Order
composedBP bool
)

// Read targets from buildpack.toml
pathToBuildpackToml := filepath.Join(path, "buildpack.toml")
if _, err := os.Stat(pathToBuildpackToml); err == nil {
buildpackCfg, err := packageConfigReader.ReadBuildpackDescriptor(pathToBuildpackToml)
if err != nil {
return nil, false, err
}
targets = buildpackCfg.Targets()
order = buildpackCfg.Order()
composedBP = len(order) > 0
}

// When composite buildpack, targets are defined in package.toml - See RFC-0128
if composedBP {
targets = bpPackageCfg.Targets
}
return targets, composedBP, nil
}

func clean(paths []string) error {
// we need to clean the buildpack.toml for each place where we copied to
if len(paths) > 0 {
for _, path := range paths {
os.Remove(path)
}
}
return nil
}
Loading

0 comments on commit cb4b368

Please sign in to comment.