-
Notifications
You must be signed in to change notification settings - Fork 37
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: lockfile, pkg, and lock v1-to-v2 auto-migration
* move all of the "package logic" to pkg.bzl
* The v2 lockfile format:
* doesn't need the fast_package_lookup dict because it's already using
a dict to store the packages.
* has the dependencies sorted so the lockfile now has stable
serialization and the diffs of the lock are actually usable and
useful to compare with the changes to the manifest.
* removes the package and dependency key from the lockfile, now it's
done via an external function (make_deb_import_key in
deb_import.bzl)
* Remove add_package_dependency from the lockfile API. Now, the package
dependencies are passed as an argument to add_package. This way, the
lockfile functionality is fully contained in lockfile.bzl and e.g. we
can remove the "consistency checks" that were only needed because
users could forget to add the dependency as a package to the lockfile.
* Ensure backwards-compatibility by internally converting lock v1 to v2.
Also, when a lock is set and it's in v1 format, there's a reminder
that encourages the users to run @repo//:lock to update the lockfile
format.- Loading branch information
Showing
9 changed files
with
220 additions
and
121 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,82 +1,119 @@ | ||
| "lock" | ||
|
|
||
| load(":util.bzl", "util") | ||
| load(":pkg.bzl", "dep", "pkg") | ||
|
|
||
| def _make_package_key(name, version, arch): | ||
| return "%s_%s_%s" % ( | ||
| util.sanitize(name), | ||
| util.sanitize(version), | ||
| arch, | ||
| ) | ||
| def __empty(): | ||
| return struct(version = 2, packages = {}) | ||
|
|
||
| def _package_key(package, arch): | ||
| return _make_package_key(package["Package"], package["Version"], arch) | ||
| def __has_package(lock, p): | ||
| return ( | ||
| p.name in lock.packages and | ||
| p.arch in lock.packages[p.name] and | ||
| p.version == lock.packages[p.name][p.arch].version | ||
| ) | ||
|
|
||
| def _add_package(lock, package, arch): | ||
| k = _package_key(package, arch) | ||
| if k in lock.fast_package_lookup: | ||
| def __add_package(lock, p, arch): | ||
| if __has_package(lock, p): | ||
| return | ||
| lock.packages.append({ | ||
| "key": k, | ||
| "name": package["Package"], | ||
| "version": package["Version"], | ||
| "url": "%s/%s" % (package["Root"], package["Filename"]), | ||
| "sha256": package["SHA256"], | ||
| "arch": arch, | ||
| "dependencies": [], | ||
| }) | ||
| lock.fast_package_lookup[k] = len(lock.packages) - 1 | ||
|
|
||
| def _add_package_dependency(lock, package, dependency, arch): | ||
| k = _package_key(package, arch) | ||
| if k not in lock.fast_package_lookup: | ||
| fail("Broken state: %s is not in the lockfile." % package["Package"]) | ||
| i = lock.fast_package_lookup[k] | ||
| lock.packages[i]["dependencies"].append(dict( | ||
| key = _package_key(dependency, arch), | ||
| name = dependency["Package"], | ||
| version = dependency["Version"], | ||
| )) | ||
|
|
||
| def _has_package(lock, name, version, arch): | ||
| key = "%s_%s_%s" % (util.sanitize(name), util.sanitize(version), arch) | ||
| return key in lock.fast_package_lookup | ||
|
|
||
| if p.name not in lock.packages: | ||
| lock.packages[p.name] = {} | ||
|
|
||
| lock.packages[p.name][p.arch] = p | ||
|
|
||
| def _add_package(lock, package, arch, dependencies_to_add = None): | ||
| dependencies_to_add = dependencies_to_add or [] | ||
|
|
||
| p = pkg.from_index(package, arch) | ||
|
|
||
| __add_package(lock, p, arch) | ||
|
|
||
| # NOTE: sorting the dependencies makes the contents | ||
| # of the lock file stable and thus they diff better | ||
| dependencies_to_add = sorted( | ||
| dependencies_to_add, | ||
| key = lambda p: (p["Package"], p["Version"]), | ||
| ) | ||
|
|
||
| for dependency in dependencies_to_add: | ||
| p_dep = pkg.from_index(dependency, arch) | ||
|
|
||
| __add_package(lock, p_dep, arch) | ||
|
|
||
| d = dep.from_index(dependency, arch) | ||
| lock.packages[p.name][p.arch].dependencies.append(d) | ||
|
|
||
| def _as_json(lock): | ||
| return json.encode_indent( | ||
| struct( | ||
| version = lock.version, | ||
| packages = lock.packages, | ||
| ), | ||
| ) | ||
|
|
||
| def _write(rctx, lock, out): | ||
| return rctx.file(out, _as_json(lock)) | ||
|
|
||
| def _create(rctx, lock): | ||
| return struct( | ||
| has_package = lambda *args, **kwargs: _has_package(lock, *args, **kwargs), | ||
| add_package = lambda *args, **kwargs: _add_package(lock, *args, **kwargs), | ||
| add_package_dependency = lambda *args, **kwargs: _add_package_dependency(lock, *args, **kwargs), | ||
| packages = lambda: lock.packages, | ||
| write = lambda out: rctx.file(out, json.encode_indent(struct(version = lock.version, packages = lock.packages))), | ||
| as_json = lambda: json.encode_indent(struct(version = lock.version, packages = lock.packages)), | ||
| packages = lock.packages, | ||
| add_package = lambda package, arch, dependencies_to_add: _add_package( | ||
| lock, | ||
| package, | ||
| arch, | ||
| dependencies_to_add, | ||
| ), | ||
| as_json = lambda: _as_json(lock), | ||
| write = lambda out: _write(rctx, lock, out), | ||
| ) | ||
|
|
||
| def _empty(rctx): | ||
| lock = struct( | ||
| version = 1, | ||
| packages = list(), | ||
| fast_package_lookup = dict(), | ||
| ) | ||
| return _create(rctx, lock) | ||
| return _create(rctx, __empty()) | ||
|
|
||
| def _from_json(rctx, content): | ||
| lock = json.decode(content) | ||
| if lock["version"] != 1: | ||
| fail("invalid lockfile version") | ||
| def _from_lock_v1(lock_content): | ||
| if lock_content["version"] != 1: | ||
| fail("Invalid lockfile version: %s" % lock_content["version"]) | ||
|
|
||
| lockv2 = __empty() | ||
|
|
||
| for package in lock_content["packages"]: | ||
| p = pkg.from_lock_v1(package) | ||
| __add_package(lockv2, p, p.arch) | ||
|
|
||
| return lockv2 | ||
|
|
||
| def _from_lock_v2(lock_content): | ||
| if lock_content["version"] != 2: | ||
| fail("Invalid lockfile version: %s" % lock_content["version"]) | ||
|
|
||
| lockv2 = __empty() | ||
|
|
||
| for archs in lock_content["packages"].values(): | ||
| for package in archs.values(): | ||
| p = pkg.from_lock(package) | ||
| __add_package(lockv2, p, p.arch) | ||
|
|
||
| return lockv2 | ||
|
|
||
| def _from_json(rctx, lock_content): | ||
| lock_content = json.decode(lock_content) | ||
|
|
||
| if lock_content["version"] == 2: | ||
| lock = _from_lock_v2(lock_content) | ||
| elif lock_content["version"] == 1: | ||
| print( | ||
| "\n\nAuto-converting lockfile format from v1 to v2. " + | ||
| "To permanently convert an existing lockfile please run " + | ||
| "`bazel run @<REPO>//:lock`\n\n", | ||
| ) | ||
|
|
||
| lock = _from_lock_v1(lock_content) | ||
| else: | ||
| fail("Invalid lockfile version: %s" % lock_content["version"]) | ||
|
|
||
| lock = struct( | ||
| version = lock["version"], | ||
| packages = lock["packages"], | ||
| fast_package_lookup = dict(), | ||
| ) | ||
| for (i, package) in enumerate(lock.packages): | ||
| lock.packages[i] = package | ||
| lock.fast_package_lookup[package["key"]] = i | ||
| return _create(rctx, lock) | ||
|
|
||
| lockfile = struct( | ||
| empty = _empty, | ||
| from_json = _from_json, | ||
| make_package_key = _make_package_key, | ||
| ) |
Oops, something went wrong.