diff --git a/dockerfiles/Makefile b/dockerfiles/Makefile index 2d4a8a3..5d666c7 100644 --- a/dockerfiles/Makefile +++ b/dockerfiles/Makefile @@ -2,7 +2,7 @@ NAMESPACE ?= ghcr.io/esgf-nimbus -build push run: REPOSITORY = $(lastword $(subst /, ,$(PWD))) +build push run readme: REPOSITORY = $(lastword $(subst /, ,$(PWD))) build: docker build $(ARGS) -t $(NAMESPACE)/$(REPOSITORY):$(VERSION) . $(if $(POST_BUILD),&& $(POST_BUILD),) @@ -18,3 +18,6 @@ changelog: bump-%: tbump --no-tag --no-push $(shell pysemver bump $* $(shell tbump current-version)) && make changelog && git add changelog.md && git commit -m "Updates changelog" + +readme: + docker run -it --rm --entrypoint python -v $(PWD):/host -w /host $(NAMESPACE)/$(REPOSITORY):$(VERSION) /host/generate.py $(ENVIRONMENT_FILE) diff --git a/dockerfiles/climate-notebook-gpu/Makefile b/dockerfiles/climate-notebook-gpu/Makefile index 04c2451..bb73d03 100644 --- a/dockerfiles/climate-notebook-gpu/Makefile +++ b/dockerfiles/climate-notebook-gpu/Makefile @@ -4,4 +4,6 @@ VERSION = 0.1.5 run: ARGS = -p 8888:8888 +ENVIRONMENT_FILE = base.yaml + include ../Makefile diff --git a/dockerfiles/climate-notebook-gpu/README.md b/dockerfiles/climate-notebook-gpu/README.md new file mode 100644 index 0000000..b2bf0cb --- /dev/null +++ b/dockerfiles/climate-notebook-gpu/README.md @@ -0,0 +1,6 @@ +# Packages + +| ML packages | +| ----------- | +| rapids 0.0.1 | +| cupy 13.2.0 | \ No newline at end of file diff --git a/dockerfiles/climate-notebook-gpu/generate.py b/dockerfiles/climate-notebook-gpu/generate.py new file mode 100644 index 0000000..dfb1075 --- /dev/null +++ b/dockerfiles/climate-notebook-gpu/generate.py @@ -0,0 +1,104 @@ +import json +import re +import subprocess +import itertools +import argparse + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("file", help="Conda environment file") + + kwargs = vars(parser.parse_args()) + + categories = get_categories(**kwargs) + + search_args = "|".join( + [f"^{x}$" for packages in categories.values() for x in packages] + ) + + output = subprocess.run( + f"mamba list {search_args!r} --json", + capture_output=True, + shell=True + ) + + output_json = json.loads(output.stdout) + + versions = { + package["name"]: package["version"] for package in output_json + } + + table = [] + categories = list(categories.items()) + + for i in range(0, len(categories), 3): + header = " | ".join([x[0] for x in categories[i:i+3]]) + separator = " | ".join([f"{'-'*len(x[0])}" for x in categories[i:i+3]]) + header = f"| {header} |\n| {separator} |" + + rows = itertools.zip_longest(*[x[1] for x in categories[i:i+3]]) + rows = [ + " | ".join( + [ + "" if y is None else f"{y} {get_version(y, versions)}" + for y in x + ] + ) for x in rows + ] + rows = "\n".join(f"| {x} |" for x in rows) + + table.append(f"{header}\n{rows}") + + table = "\n".join(table) + + data = f"""# Packages + +{table}""" + + with open("README.md", "w") as fd: + fd.write(data) + + +def get_version(package, versions): + m = re.search(r"([^<>=]*)(?:(?:<|>|<=|>=|==).*)?", package) + + if m is None: + raise Exception("Could not parse ", package) + + name = m.group(1) + + return versions[name] + + +def get_categories(file, **_): + with open(file) as fd: + lines = fd.readlines() + + category = None + packages = {} + + for line in lines: + if "#" in line: + m = re.search("# (.*)(?: packages)?", line) + + if m is None: + continue + + category = m.group(1) + elif category != "ignore" and category is not None: + m = re.search("- (.*)", line) + + if m is None: + continue + + if category in packages: + packages[category].append(m.group(1)) + else: + packages[category] = [m.group(1)] + + return packages + + +if __name__ == "__main__": + main() diff --git a/dockerfiles/climate-notebook/Makefile b/dockerfiles/climate-notebook/Makefile index 44b1ee6..bb5cc5d 100644 --- a/dockerfiles/climate-notebook/Makefile +++ b/dockerfiles/climate-notebook/Makefile @@ -12,4 +12,6 @@ export-env: ENTRYPOINT = /bin/bash export-env: COMMAND = -c "cp /base.yaml /conda-envs/" export-env: run +ENVIRONMENT_FILE = base.yaml + include ../Makefile diff --git a/dockerfiles/climate-notebook/README.md b/dockerfiles/climate-notebook/README.md new file mode 100644 index 0000000..11271b8 --- /dev/null +++ b/dockerfiles/climate-notebook/README.md @@ -0,0 +1,16 @@ +# Packages + +| Data packages | Climate packages | Plotting packages | +| ------------- | ---------------- | ----------------- | +| xarray 2024.6.0 | xcdat 0.7.0 | matplotlib 3.8.4 | +| xarray-datatree 0.0.14 | xesmf 0.8.5 | holoviews 1.17.1 | +| pandas 2.2.2 | xgcm 0.8.1 | hvplot 0.10.0 | +| flox 0.9.8 | pcmdi_metrics 3.4.1 | geoviews 1.9.6 | +| netcdf4 1.6.5 | gsw-xarray 0.4.0 | seaborn 0.13.2 | +| s3fs 2024.6.0 | | cartopy 0.23.0 | +| zarr 2.18.2 | | datashader 0.16.2 | +| Computing packages | Discovery packages | +| ------------------ | ------------------ | +| dask-gateway 2024.1.0 | intake-esm 2024.2.6 | +| | intake-stac 0.4.0 | +| | intake-xarray 0.7.0 | \ No newline at end of file diff --git a/dockerfiles/climate-notebook/generate.py b/dockerfiles/climate-notebook/generate.py new file mode 100644 index 0000000..dfb1075 --- /dev/null +++ b/dockerfiles/climate-notebook/generate.py @@ -0,0 +1,104 @@ +import json +import re +import subprocess +import itertools +import argparse + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("file", help="Conda environment file") + + kwargs = vars(parser.parse_args()) + + categories = get_categories(**kwargs) + + search_args = "|".join( + [f"^{x}$" for packages in categories.values() for x in packages] + ) + + output = subprocess.run( + f"mamba list {search_args!r} --json", + capture_output=True, + shell=True + ) + + output_json = json.loads(output.stdout) + + versions = { + package["name"]: package["version"] for package in output_json + } + + table = [] + categories = list(categories.items()) + + for i in range(0, len(categories), 3): + header = " | ".join([x[0] for x in categories[i:i+3]]) + separator = " | ".join([f"{'-'*len(x[0])}" for x in categories[i:i+3]]) + header = f"| {header} |\n| {separator} |" + + rows = itertools.zip_longest(*[x[1] for x in categories[i:i+3]]) + rows = [ + " | ".join( + [ + "" if y is None else f"{y} {get_version(y, versions)}" + for y in x + ] + ) for x in rows + ] + rows = "\n".join(f"| {x} |" for x in rows) + + table.append(f"{header}\n{rows}") + + table = "\n".join(table) + + data = f"""# Packages + +{table}""" + + with open("README.md", "w") as fd: + fd.write(data) + + +def get_version(package, versions): + m = re.search(r"([^<>=]*)(?:(?:<|>|<=|>=|==).*)?", package) + + if m is None: + raise Exception("Could not parse ", package) + + name = m.group(1) + + return versions[name] + + +def get_categories(file, **_): + with open(file) as fd: + lines = fd.readlines() + + category = None + packages = {} + + for line in lines: + if "#" in line: + m = re.search("# (.*)(?: packages)?", line) + + if m is None: + continue + + category = m.group(1) + elif category != "ignore" and category is not None: + m = re.search("- (.*)", line) + + if m is None: + continue + + if category in packages: + packages[category].append(m.group(1)) + else: + packages[category] = [m.group(1)] + + return packages + + +if __name__ == "__main__": + main()