forked from bazelbuild/rules_docker
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement new interface for legacy intermediate format layouts (bazel…
…build#954) * Implement new interface for MM format images * Add documentation and reformat code * Added copyright headers * Add header in the BUILD file * Fix BUILD file header * Restyle with go tools * Refactor files paths usages * Add lock protection for raw manifest * Implement pusher to incorporate new legacy format * Add more comments * Add comments for legacy reader * Reformat comments * Refactor code after review * Fixed typo in code * Refactor code after comments * Refactor input file directory trimming * Refactor src path checking
- Loading branch information
1 parent
a84ff53
commit b190dfc
Showing
7 changed files
with
325 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,155 @@ | ||
| /// Copyright 2015 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| ////////////////////////////////////////////////////////////////////// | ||
| // Image for intermediate format used in python containerregistry. | ||
| // Adopted from go-containerregistry's layout.image implementation with modification to understand rules_docker's legacy intermediate format. | ||
| // Uses the go-containerregistry API as backend. | ||
|
|
||
| package compat | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
| "sync" | ||
|
|
||
| v1 "github.com/google/go-containerregistry/pkg/v1" | ||
| "github.com/google/go-containerregistry/pkg/v1/partial" | ||
| "github.com/google/go-containerregistry/pkg/v1/types" | ||
| ) | ||
|
|
||
| // legacyImage is the image in legacy intermediate format. Implements v1.Image, and its implementation is very similar to layout.layoutImage. | ||
| type legacyImage struct { | ||
| // path is the path to the directory containing the legacy image. | ||
| path string | ||
| // digest is the sha256 hash for this image. | ||
| digest v1.Hash | ||
| // manifestLock protects rawManifest. | ||
| manifestLock sync.Mutex | ||
| // rawManifest is the raw bytes of manifest.json file. | ||
| rawManifest []byte | ||
| } | ||
|
|
||
| var _ partial.CompressedImageCore = (*legacyImage)(nil) | ||
|
|
||
| // MediaType of this image's manifest from manifest.json. | ||
| func (li *legacyImage) MediaType() (types.MediaType, error) { | ||
| manifest, err := li.Manifest() | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
|
|
||
| if manifest.MediaType != types.OCIManifestSchema1 && manifest.MediaType != types.DockerManifestSchema2 { | ||
| return "", fmt.Errorf("unexpected media type for %v: %s", li.digest, manifest.MediaType) | ||
| } | ||
|
|
||
| return manifest.MediaType, nil | ||
| } | ||
|
|
||
| // Parses manifest.json into Manifest object. Implements WithManifest for partial.Blobset. | ||
| func (li *legacyImage) Manifest() (*v1.Manifest, error) { | ||
| return partial.Manifest(li) | ||
| } | ||
|
|
||
| // RawManifest returns the serialized bytes of manifest.json metadata. | ||
| func (li *legacyImage) RawManifest() ([]byte, error) { | ||
| li.manifestLock.Lock() | ||
| defer li.manifestLock.Unlock() | ||
|
|
||
| if li.rawManifest != nil { | ||
| return li.rawManifest, nil | ||
| } | ||
|
|
||
| // Read and store raw manifest.json file from src directory. | ||
| b, err := ioutil.ReadFile(filepath.Join(li.path, manifestFile)) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| li.rawManifest = b | ||
| return li.rawManifest, nil | ||
| } | ||
|
|
||
| // RawConfigFile returns the serialized bytes of config.json metadata. | ||
| func (li *legacyImage) RawConfigFile() ([]byte, error) { | ||
| return ioutil.ReadFile(filepath.Join(li.path, configFile)) | ||
| } | ||
|
|
||
| // LayerByDigest returns a Layer for interacting with a particular layer of the image, looking it up by "digest" (the compressed hash). | ||
| // We assume the layer files are named in the format of e.g., 000.tar.gz in this path, following the order they appear in manifest.json. | ||
| func (li *legacyImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { | ||
| manifest, err := li.Manifest() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // The config is a layer in some cases. | ||
| if h == manifest.Config.Digest { | ||
| return partial.CompressedLayer(&compressedBlob{ | ||
| path: li.path, | ||
| desc: manifest.Config, | ||
| filename: "config.json", | ||
| }), nil | ||
| } | ||
|
|
||
| for i, desc := range manifest.Layers { | ||
| if h == desc.Digest { | ||
| switch desc.MediaType { | ||
| case types.OCILayer, types.DockerLayer: | ||
| return partial.CompressedToLayer(&compressedBlob{ | ||
| path: li.path, | ||
| desc: desc, | ||
| filename: layerFilename(i), | ||
| }) | ||
| default: | ||
| // TODO: We assume everything is a compressed blob, but that might not be true. | ||
| // TODO: Handle foreign layers. | ||
| return nil, fmt.Errorf("unexpected media type: %v for layer: %v", desc.MediaType, desc.Digest) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil, fmt.Errorf("could not find layer in image: %s", h) | ||
| } | ||
|
|
||
| type compressedBlob struct { | ||
| // path of this compressed blob. | ||
| path string | ||
| // desc is the descriptor of this compressed blob. | ||
| desc v1.Descriptor | ||
| // filename is the filename of this blob at the directory. | ||
| filename string | ||
| } | ||
|
|
||
| // The digest of this compressedBlob. | ||
| func (b *compressedBlob) Digest() (v1.Hash, error) { | ||
| return b.desc.Digest, nil | ||
| } | ||
|
|
||
| // Return and open a the layer file at path. | ||
| func (b *compressedBlob) Compressed() (io.ReadCloser, error) { | ||
| return os.Open(filepath.Join(b.path, b.filename)) | ||
| } | ||
|
|
||
| // The size of this compressedBlob. | ||
| func (b *compressedBlob) Size() (int64, error) { | ||
| return b.desc.Size, nil | ||
| } | ||
|
|
||
| // The media type of this compressedBlob. | ||
| func (b *compressedBlob) MediaType() (types.MediaType, error) { | ||
| return b.desc.MediaType, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| /// Copyright 2015 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| ////////////////////////////////////////////////////////////////////// | ||
| // Path utils used for legacy image layout outputted by python containerregistry. | ||
| // Uses the go-containerregistry API as backend. | ||
|
|
||
| package compat | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| ) | ||
|
|
||
| // Expected metadata files in legacy layout. | ||
| const ( | ||
| manifestFile = "manifest.json" | ||
| configFile = "config.json" | ||
| digestFile = "digest" | ||
| ) | ||
|
|
||
| // Return the filename for layer at index i in the layers array in manifest.json. | ||
| // Assume the layers are padded to three digits, e.g., the first layer is named 000.tar.gz. | ||
| func layerFilename(i int) string { | ||
| return fmt.Sprintf("%03d.tar.gz", i) | ||
| } | ||
|
|
||
| // Naively validates a legacy intermediate layout at <path> by checking if digest, config.json, and manifest.json all exist. | ||
| func isValidLegacylayout(path string) (bool, error) { | ||
| if _, err := os.Stat(filepath.Join(path, manifestFile)); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| if _, err := os.Stat(filepath.Join(path, configFile)); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| if _, err := os.Stat(filepath.Join(path, digestFile)); err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| return true, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| // Copyright 2015 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| ////////////////////////////////////////////////////////////////////// | ||
| // Reads an legacy image layout on disk. | ||
| package compat | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io/ioutil" | ||
| "path/filepath" | ||
|
|
||
| v1 "github.com/google/go-containerregistry/pkg/v1" | ||
| "github.com/google/go-containerregistry/pkg/v1/partial" | ||
| "github.com/google/go-containerregistry/pkg/v1/validate" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| // Read returns a docker image referenced by the legacy intermediate layout at src. The image index should have been outputted by container_pull. | ||
| // NOTE: this only reads index with a single image. | ||
| func Read(src string) (v1.Image, error) { | ||
| _, err := isValidLegacylayout(src) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "invalid legacy layout at %s, requires manifest.json, config.json and digest files", src) | ||
| } | ||
|
|
||
| digest, err := getManifestDigest(src) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "unable to get manifest digest from %s", src) | ||
| } | ||
|
|
||
| // Constructs and validates a v1.Image object. | ||
| legacyImg := &legacyImage{ | ||
| path: src, | ||
| digest: digest, | ||
| } | ||
|
|
||
| img, err := partial.CompressedToImage(legacyImg) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "unable to load image with digest %s obtained from the manifest at %s", digest, src) | ||
| } | ||
|
|
||
| if err := validate.Image(img); err != nil { | ||
| return nil, errors.Wrapf(err, "unable to load image with digest %s due to invalid legacy layout format from %s", digest, src) | ||
| } | ||
|
|
||
| return img, nil | ||
| } | ||
|
|
||
| // Get the hash of the image to read at <path> from digest file. | ||
| func getManifestDigest(path string) (v1.Hash, error) { | ||
| // We expect a file named digest that stores the manifest's hash formatted as sha256:{Hash} in this directory. | ||
| digest, err := ioutil.ReadFile(filepath.Join(path, digestFile)) | ||
| if err != nil { | ||
| return v1.Hash{}, fmt.Errorf("failed to locate SHA256 digest file for image manifest: %v", err) | ||
| } | ||
|
|
||
| return v1.NewHash(string(digest)) | ||
| } |
Oops, something went wrong.