From 88747a475e5b4005993e28bafd19a09d60ff821e Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 5 Sep 2022 17:41:08 -0500
Subject: [PATCH 001/161] very much wip

---
 .idea/inspectionProfiles/Project_Default.xml |  15 --
 .idea/runConfigurations/ce_install.xml       |   2 +-
 Makefile                                     |   4 +-
 bin/cefs                                     |   1 +
 bin/lib/ce_install.py                        |  80 ++++++
 bin/lib/cefs.py                              | 243 +++++++++++++++++++
 bin/lib/installation.py                      |   2 -
 bin/test/cefs_test.py                        | 105 ++++++++
 8 files changed, 432 insertions(+), 20 deletions(-)
 delete mode 100644 .idea/inspectionProfiles/Project_Default.xml
 create mode 120000 bin/cefs
 create mode 100644 bin/lib/cefs.py
 create mode 100644 bin/test/cefs_test.py

diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
deleted file mode 100644
index cdc00f963..000000000
--- a/.idea/inspectionProfiles/Project_Default.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<component name="InspectionProjectProfileManager">
-  <profile version="1.0">
-    <option name="myName" value="Project Default" />
-    <inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
-      <option name="ourVersions">
-        <value>
-          <list size="2">
-            <item index="0" class="java.lang.String" itemvalue="2.7" />
-            <item index="1" class="java.lang.String" itemvalue="3.11" />
-          </list>
-        </value>
-      </option>
-    </inspection_tool>
-  </profile>
-</component>
\ No newline at end of file
diff --git a/.idea/runConfigurations/ce_install.xml b/.idea/runConfigurations/ce_install.xml
index 7b61ea0bc..832508bcc 100644
--- a/.idea/runConfigurations/ce_install.xml
+++ b/.idea/runConfigurations/ce_install.xml
@@ -13,7 +13,7 @@
     <option name="ADD_SOURCE_ROOTS" value="true" />
     <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
     <option name="SCRIPT_NAME" value="$PROJECT_DIR$/bin/lib/ce_install.py" />
-    <option name="PARAMETERS" value="squash x86/gcc" />
+    <option name="PARAMETERS" value="--cache /tmp/ce-install-cache buildroot clang 14.0.0" />
     <option name="SHOW_COMMAND_LINE" value="false" />
     <option name="EMULATE_TERMINAL" value="false" />
     <option name="MODULE_MODE" value="false" />
diff --git a/Makefile b/Makefile
index 0afe5422d..58fc26eeb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 .NOTPARALLEL:
 
-PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin bash -c "command -v python3.10 || command -v python3.9 || command -v python3.8 || echo .python3.8-not-found")
+PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin bash -c "command -v python3.10 || command -v python3.9 || echo .python3.9-not-found")
 VIRTUALENV?=.env
 VIRTUALENV_DONE:=$(VIRTUALENV)/.done
 export PYTHONPATH=$(CURDIR)/bin
@@ -12,7 +12,7 @@ help: # with thanks to Ben Rady
 PACKER ?= ../packer
 
 $(PYTHON):
-	@echo "Python 3.8, 3.9 or 3.10 not found on path. Please install (sudo apt install python3.8 python3.8-venv or similar)"
+	@echo "Python 3.9 or 3.10 not found on path. Please install (sudo apt install python3.9 python3.9-venv or similar)"
 	@exit 1
 
 config.json: make_json.py | $(PYTHON)
diff --git a/bin/cefs b/bin/cefs
new file mode 120000
index 000000000..ae249b7dd
--- /dev/null
+++ b/bin/cefs
@@ -0,0 +1 @@
+scripts/python_shim
\ No newline at end of file
diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py
index a715d2ab0..a63812fb3 100644
--- a/bin/lib/ce_install.py
+++ b/bin/lib/ce_install.py
@@ -10,12 +10,14 @@
 from dataclasses import dataclass
 from functools import partial
 from pathlib import Path
+from tempfile import TemporaryDirectory
 from typing import List, Optional, Tuple
 
 import click
 import yaml
 
 from lib.amazon_properties import get_properties_compilers_and_libraries
+from lib.cefs import CefsImage, SquashFsCreator
 from lib.config_safe_loader import ConfigSafeLoader
 from lib.installation import InstallationContext, installers_for, Installable
 from lib.library_yaml import LibraryYaml
@@ -339,6 +341,84 @@ def squash(context: CliContext, filter_: List[str], force: bool, image_dir: Path
             installable.squash_to(destination)
 
 
+CEFS_ROOT = Path("/cefs")
+
+
+@cli.command()
+@click.pass_obj
+@click.option("--force", is_flag=True, help="Force even if would otherwise skip")
+@click.option(
+    "--cefs-root",
+    default=Path("/cefs"),
+    metavar="CEFS_ROOT",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Install or assume cefs is installed at CEFS_ROOT",
+    show_default=True,
+)
+@click.option(
+    "--squash-image-root",
+    default=Path("/opt/cefs-images"),
+    metavar="IMAGE_DIR",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Store or look for squashfs images in IMAGE_DIR",
+    show_default=True,
+)
+@click.argument("filter_", metavar="FILTER", nargs=-1)
+def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image_root: Path, cefs_root: Path):
+    """Squash all things matching to a single root image."""
+
+    # check for things already installed in the _current_ root?
+
+    installation_context = context.installation_context
+
+    if not installation_context.destination.is_symlink():
+        click.echo(f"Destination {installation_context.destination} is not a CEFS root symlink!")
+        sys.exit(1)
+    cefs_root_link = installation_context.destination.readlink()
+    while cefs_root_link.is_symlink():
+        cefs_root_link = cefs_root_link.readlink()
+    if not cefs_root_link.is_relative_to(cefs_root):
+        click.echo(
+            f"Destination {installation_context.destination} is not a CEFS root symlink "
+            f"({cefs_root_link} not relative to {cefs_root})!"
+        )
+        sys.exit(1)
+    current_image = CefsImage(cefs_root=cefs_root, directory=installation_context.destination)
+    current_image.add_metadata(f"Information read from root image {cefs_root_link}")
+    for installable in context.get_installables(filter_):
+        if force or installable.is_installed():
+            dest_path = installation_context.destination / installable.install_path
+            if not dest_path.is_symlink():
+                _LOGGER.error("Found an installable that wasn't a symlink: %s", dest_path)
+                sys.exit(1)
+
+    install_creator = SquashFsCreator(squash_image_root=squash_image_root, cefs_root=cefs_root)
+    with install_creator as tmp_path:
+        installation_context._staging_root = tmp_path / "staging"
+        installation_context.destination = tmp_path
+        _LOGGER.info("Installing everything to a temp dir")
+        for installable in context.get_installables(filter_):
+            if Path(installable.install_path) not in current_image.catalog:
+                installable.install()
+                current_image.add_metadata(f"Installing {installable.install_path} from {installable}")
+
+    new_squashfs_cefs = install_creator.cefs_path
+    for installable in context.get_installables(filter_):
+        if Path(installable.install_path) not in current_image.catalog:
+            current_image.link_path(Path(installable.install_path), new_squashfs_cefs / installable.install_path)
+
+    _LOGGER.info("Building new root fs")
+    root_creator = SquashFsCreator(squash_image_root=squash_image_root, cefs_root=cefs_root)
+    with root_creator as tmp_path:
+        current_image.render_to(tmp_path)
+
+    new_squashfs_cefs = root_creator.cefs_path
+    print(f"Output to {new_squashfs_cefs} replacing {cefs_root_link}")
+    # TODO keep track of previous? then we can "undo"
+    Path("/home/mgodbolt/ce").unlink(missing_ok=True)
+    Path("/home/mgodbolt/ce").symlink_to(new_squashfs_cefs)
+
+
 @cli.command()
 @click.pass_obj
 @click.option(
diff --git a/bin/lib/cefs.py b/bin/lib/cefs.py
new file mode 100644
index 000000000..4258e50ab
--- /dev/null
+++ b/bin/lib/cefs.py
@@ -0,0 +1,243 @@
+import logging
+import os
+import shutil
+import sys
+import subprocess
+from pathlib import Path
+from tempfile import mkdtemp, NamedTemporaryFile, TemporaryDirectory
+from typing import Optional, Mapping, List, Dict
+from dataclasses import dataclass
+
+import click
+
+_LOGGER = logging.getLogger(__name__)
+METADATA_FILENAME = "metadata.txt"
+
+
+@dataclass(frozen=True)
+class CliContext:
+    cefs_root: Path
+    squash_image_root: Path
+
+
+@click.group()
+@click.option(
+    "--cefs-root",
+    default=Path("/cefs"),
+    metavar="CEFS_ROOT",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Install or assume cefs is installed at CEFS_ROOT",
+    show_default=True,
+)
+@click.option(
+    "--squash-image-root",
+    default=Path("/opt/cefs-images"),
+    metavar="IMAGE_DIR",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Store or look for squashfs images in IMAGE_DIR",
+    show_default=True,
+)
+@click.option("--debug/--no-debug", help="Turn on debugging")
+@click.option("--log-to-console", is_flag=True, help="Log output to console, even if logging to a file is requested")
+@click.option("--log", metavar="LOGFILE", help="Log to LOGFILE", type=click.Path(dir_okay=False, writable=True))
+@click.pass_context
+def cli(
+    ctx: click.Context,
+    cefs_root: Path,
+    squash_image_root: Path,
+    debug: bool,
+    log_to_console: bool,
+    log: Optional[str],
+):
+    """Administrate the Compiler Explorer File System (cefs)."""
+    formatter = logging.Formatter(fmt="%(asctime)s %(name)-15s %(levelname)-8s %(message)s")
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG if debug else logging.INFO)
+    if log:
+        file_handler = logging.FileHandler(log)
+        file_handler.setFormatter(formatter)
+        root_logger.addHandler(file_handler)
+    if not log or log_to_console:
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setFormatter(formatter)
+        root_logger.addHandler(console_handler)
+    ctx.obj = CliContext(cefs_root=cefs_root, squash_image_root=squash_image_root)
+
+
+@cli.command
+@click.pass_obj
+def install(context: CliContext):
+    """Install the CEFS. Needs to be run as root."""
+    if os.geteuid() != 0:
+        click.echo("Installing cefs needs root privileges.")
+        sys.exit(1)
+    if not Path("/etc/auto.master.d").is_dir():
+        click.echo("Installing cefs needs autofs. Please install manually using e.g. `apt install autofs`")
+        sys.exit(1)
+    click.echo("Writing config files...")
+    auto_cefs_config_file = Path("/etc/auto.cefs")
+    auto_cefs_config_file.write_text(
+        f"* -fstype=squashfs,loop,nosuid,nodev,ro :{context.squash_image_root}/&.sqfs\n", encoding="utf-8"
+    )
+    Path("/etc/auto.master.d/cefs.autofs").write_text(
+        f"{context.cefs_root} {auto_cefs_config_file}  --negative-timeout 1", encoding="utf-8"
+    )
+    if not context.squash_image_root.is_dir():
+        click.echo(
+            f"Creating {context.squash_image_root} squash image root.\n"
+            f"By default this will be writable BY ALL USERS, but is sticky like /tmp. This is to make it usable\n"
+            f"by unprivileged users. You are responsible for making sure that's ok, and changing the directory\n"
+            f"permissions as appropriate."
+        )
+        context.squash_image_root.mkdir(parents=True)
+        context.squash_image_root.chmod(0o1777)
+    click.echo("Restarting autofs")
+    subprocess.check_call(["service", "autofs", "restart"])
+    click.echo(
+        f"CEFS installed with a root at {context.cefs_root} and squash image root at {context.squash_image_root}."
+    )
+    click.echo(
+        f"Note that the root {context.cefs_root} will appear empty, "
+        f"but will automatically mount on demand when required."
+    )
+
+
+class SquashFsCreator:
+    def __init__(self, *, squash_image_root: Path, cefs_root: Path):
+        self._squash_image_root = squash_image_root
+        self._cefs_root = cefs_root
+        self._sha = None
+
+    def _close(self):
+        with TemporaryDirectory(prefix="ce-squash-builder") as tmp_dir:
+            tmp_sqfs = Path(tmp_dir) / "temp.sqfs"
+            subprocess.check_call(
+                [
+                    "/usr/bin/mksquashfs",
+                    str(self._path),
+                    str(tmp_sqfs),
+                    "-all-root",
+                    "-root-mode",
+                    "755",
+                    "-progress",
+                    "-comp",
+                    "zstd",
+                ]
+            )
+            self._sha, _filename = subprocess.check_output(["/usr/bin/shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            if not self.image.exists():
+                _LOGGER.info("New squashfs image: %s", self.image)
+                tmp_sqfs.replace(self.image)
+            else:
+                _LOGGER.info("Existing: %s", self.image)
+                tmp_sqfs.unlink()
+
+    @property
+    def image(self) -> Path:
+        assert self._sha is not None
+        return self._squash_image_root / f"{self._sha}.sqfs"
+
+    @property
+    def cefs_path(self) -> Path:
+        assert self._sha is not None
+        return self._cefs_root / self._sha
+
+    def __enter__(self):
+        self._path = Path(mkdtemp(prefix="ce-install-temp"))
+        return self._path
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            if not exc_type:
+                self._close()
+        finally:
+            subprocess.check_call(["chmod", "-R", "u+w", str(self._path)])
+            shutil.rmtree(self._path, ignore_errors=True)
+
+
+class BadCefsRoot(RuntimeError):
+    pass
+
+
+class BadCefsLink(RuntimeError):
+    pass
+
+
+class CefsImage:
+    """
+    Holds information about a CEFS root image, which is a directory full of symlinks to
+    other CEFS images, and some metadata.
+    """
+
+    def __init__(self, *, cefs_root: Path, directory: Optional[Path] = None):
+        self._cefs_root = cefs_root.resolve(strict=True)
+        self._catalog: Dict[Path, Path] = {}
+        self._metadata: List[str] = []
+
+        if directory:
+            self._recurse_find_paths(directory, directory)
+            _LOGGER.info("CEFS image at %s has %d entries", directory, len(self.catalog))
+
+    def _recurse_find_paths(self, path: Path, root_path: Path):
+        for entry in path.iterdir():
+            relative = entry.relative_to(root_path)
+            if entry.is_symlink():
+                link = entry.readlink()
+                if not link.is_relative_to(self._cefs_root):
+                    raise BadCefsRoot(f"Found a symlink that wasn't a symlink to cefs: {entry} links to {link}")
+                _LOGGER.debug("Found existing %s -> %s", relative, link)
+                self._catalog[relative] = link
+            elif entry.is_dir():
+                self._recurse_find_paths(entry, root_path)
+            elif entry.is_file():
+                if relative == Path(METADATA_FILENAME):
+                    self._metadata = entry.read_text(encoding="utf-8").splitlines(keepends=False)
+                else:
+                    raise BadCefsRoot(f"Found an unexpected file: {entry}")
+            else:
+                raise BadCefsRoot(f"Found an unexpected entry: {entry}")
+
+    def add_metadata(self, metadata: str) -> None:
+        self._metadata.append(metadata)
+
+    def link_path(self, subdir: Path, cefs_link: Path):
+        if not cefs_link.is_relative_to(self._cefs_root):
+            raise BadCefsLink(f"Link is not relative to cefs: {cefs_link}")
+        self._catalog[subdir] = cefs_link
+
+    @property
+    def catalog(self) -> Mapping[Path, Path]:
+        return self._catalog
+
+    @property
+    def metadata(self) -> List[str]:
+        return self._metadata
+
+    def render_to(self, destination: Path) -> None:
+        destination.mkdir(parents=True, exist_ok=True)
+        (destination / METADATA_FILENAME).write_text("\n".join(self._metadata) + "\n", encoding="utf-8")
+        for entry, dest in self.catalog.items():
+            source_dir = destination / entry
+            source_dir.parent.mkdir(parents=True, exist_ok=True)
+            source_dir.symlink_to(dest)
+
+
+# TODO how to start the whole thing off? make an empty root and manually symlink it in position?
+# TODO cases where we have a GIANT image that only exists because one dir is in it
+#  ie repacking and refreshing old images
+# TODO consolidation in general! aim is to have very few layers, after all.
+# TODO delete old things
+# TODO keep track of a symlink and its history /opt/compiler-explorer --> /opt/compiler-explorer.history?
+# TODO better naming of dirs/manifest for each so we know who and why created each layer?
+# TODO should we separate out root squashfs images from data?
+# TODO detect cefs roots in `install` and redirect to buildroot?
+# TODO delete/etc
+# TODO handle old symlinks of trunk (and do trunk etc!)
+
+
+def main():
+    cli(prog_name="cefs")  # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bin/lib/installation.py b/bin/lib/installation.py
index cf3cabe84..44a3cf35b 100644
--- a/bin/lib/installation.py
+++ b/bin/lib/installation.py
@@ -690,8 +690,6 @@ def install(self) -> bool:
             self.stage(staging)
             if self.subdir:
                 self.install_context.make_subdir(self.subdir)
-            elif self.install_path:
-                self.install_context.make_subdir(self.install_path)
 
             self.install_context.move_from_staging(staging, self.untar_dir, self.install_path)
             return True
diff --git a/bin/test/cefs_test.py b/bin/test/cefs_test.py
new file mode 100644
index 000000000..96c6d2cf5
--- /dev/null
+++ b/bin/test/cefs_test.py
@@ -0,0 +1,105 @@
+from pathlib import Path
+
+import pytest
+
+from lib.cefs import CefsImage, METADATA_FILENAME, BadCefsRoot
+
+
+@pytest.fixture
+def cefs_root(tmp_path) -> Path:
+    result = tmp_path / "some-cefs-root"
+    result.mkdir()
+    return result
+
+
+@pytest.fixture
+def image_root(tmp_path) -> Path:
+    result = tmp_path / "some_image"
+    result.mkdir()
+    return result
+
+
+@pytest.fixture
+def dest_root(tmp_path) -> Path:
+    result = tmp_path / "dest"
+    result.mkdir()
+    return result
+
+
+def test_cefs_root_should_handle_no_initial_path(cefs_root: Path):
+    root = CefsImage(cefs_root=cefs_root)
+    assert not root.catalog
+    assert not root.metadata
+
+
+def test_cefs_root_should_handle_empty_dirs(cefs_root: Path, image_root: Path):
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    assert not root.catalog
+    assert not root.metadata
+
+
+def test_cefs_can_read_existing_images(cefs_root: Path, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_root / "thing")
+    (image_root / "some" / "sub").mkdir(parents=True)
+    (image_root / "some" / "sub" / "directory").symlink_to(cefs_root / "thing" / "in" / "subdir")
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    assert root.catalog == {
+        Path("thing"): cefs_root / "thing",
+        Path("some") / "sub" / "directory": cefs_root / "thing" / "in" / "subdir",
+    }
+    assert not root.metadata
+
+
+def test_cefs_can_read_existing_images_metadata(cefs_root: Path, image_root: Path, dest_root: Path):
+    (image_root / METADATA_FILENAME).write_text("Meta\nData\nFTW\n", encoding="utf-8")
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    assert root.metadata == ["Meta", "Data", "FTW"]
+
+
+def test_cefs_is_upset_by_links_to_non_cefs(cefs_root: Path, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(dest_root)  # just a valid path not in cefs
+    with pytest.raises(BadCefsRoot, match="Found a symlink that wasn't a symlink to cefs"):
+        CefsImage(directory=image_root, cefs_root=cefs_root)
+
+
+def test_cefs_is_upset_by_random_files(cefs_root: Path, image_root: Path, dest_root: Path):
+    (image_root / "thing.txt").write_text("I am a mongoose", encoding="utf-8")
+    with pytest.raises(BadCefsRoot, match="Found an unexpected file"):
+        CefsImage(directory=image_root, cefs_root=cefs_root)
+
+
+def test_cefs_can_add_metadata(cefs_root: Path, image_root: Path):
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    root.add_metadata("I am metadata")
+    assert root.metadata == ["I am metadata"]
+
+
+def test_cefs_can_render_metadata(cefs_root: Path, image_root: Path, dest_root: Path):
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    root.add_metadata("Line 1")
+    root.add_metadata("Line 2")
+    root.add_metadata("Line 3")
+    root.render_to(dest_root)
+    assert (dest_root / METADATA_FILENAME).read_text(encoding="utf-8") == "Line 1\nLine 2\nLine 3\n"
+
+
+def test_cefs_can_render_existing_links(cefs_root: Path, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_root / "thing")
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    root.render_to(dest_root)
+    assert (dest_root / "thing").readlink() == Path(cefs_root / "thing")
+
+
+def test_cefs_can_render_new_links(cefs_root: Path, image_root: Path, dest_root: Path):
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    root.link_path(Path("something"), cefs_root / "something")
+    root.render_to(dest_root)
+    assert (dest_root / "something").readlink() == Path(cefs_root / "something")
+
+
+def test_cefs_can_replace_existing_links(cefs_root: Path, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_root / "thing")
+    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    root.link_path(Path("thing"), cefs_root / "new")
+    root.render_to(dest_root)
+    assert (dest_root / "thing").readlink() == Path(cefs_root / "new")

From 5937368b93116915863f2b7e01fb4b5a49716167 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 5 Sep 2022 18:07:42 -0500
Subject: [PATCH 002/161] some docs

---
 bin/lib/ce_install.py |  9 ++++---
 bin/lib/cefs.md       | 62 +++++++++++++++++++++++++++++++++++++++++++
 bin/lib/cefs.py       | 16 ++++++++++-
 3 files changed, 83 insertions(+), 4 deletions(-)
 create mode 100644 bin/lib/cefs.md

diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py
index a63812fb3..f22a1046b 100644
--- a/bin/lib/ce_install.py
+++ b/bin/lib/ce_install.py
@@ -374,8 +374,10 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
     if not installation_context.destination.is_symlink():
         click.echo(f"Destination {installation_context.destination} is not a CEFS root symlink!")
         sys.exit(1)
-    cefs_root_link = installation_context.destination.readlink()
+    root_link_to_update = installation_context.destination
+    cefs_root_link = root_link_to_update.readlink()
     while cefs_root_link.is_symlink():
+        root_link_to_update = cefs_root_link
         cefs_root_link = cefs_root_link.readlink()
     if not cefs_root_link.is_relative_to(cefs_root):
         click.echo(
@@ -414,9 +416,10 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
 
     new_squashfs_cefs = root_creator.cefs_path
     print(f"Output to {new_squashfs_cefs} replacing {cefs_root_link}")
+
     # TODO keep track of previous? then we can "undo"
-    Path("/home/mgodbolt/ce").unlink(missing_ok=True)
-    Path("/home/mgodbolt/ce").symlink_to(new_squashfs_cefs)
+    root_link_to_update.unlink(missing_ok=True)
+    root_link_to_update.symlink_to(new_squashfs_cefs)
 
 
 @cli.command()
diff --git a/bin/lib/cefs.md b/bin/lib/cefs.md
new file mode 100644
index 000000000..d408fa160
--- /dev/null
+++ b/bin/lib/cefs.md
@@ -0,0 +1,62 @@
+## CEFS
+
+The Compiler Explorer File System manages a read-only, layered, hierarchy of squashfs images.
+It's designed for compilers and libraries in Compiler Explorer, but any read-only content would work. It's not really a file system, either. So it's a terrible name. But hey!
+
+### Why!
+
+We use immutable squashfs images on NFS for the compilers and libraries. That's great, fast performance, great caching behaviour. But, mounting each compiler squashfs individually is slow for system startup (~1m for ~1000). We ideally want fewer squashfs mounts, but with the ability to "overlay" changes (like daily builds) without totally rebuilding everything every time.
+
+### Overview
+
+There is an autofs mount at `/cefs` which automounts squashfs images of the form `/cefs/ID` to `/opt/cefs-images/ID.sqfs`. The filesystem root is a symlink to a squashfs image in cefs. The root itself is a number of symlinks - one per managed directory - to other locations in cefs.
+
+As an example:
+
+- `/opt/compiler-explorer` would be the root, and is a symlink to `/cefs/SOME-ROOT-ID`.
+- `/cefs/SOME_ROOT_ID` maps to a squashfs image `/opt/cefs-images/SOME_ROOT_ID.sqfs`.
+- `/opt/cefs-images/SOME_ROOT_ID.sqfs` contains symlinks for each compiler maybe something like:
+  - `gcc-12.1.0` -> `/cefs/SOME_INSTALL_ID/gcc-12.1.0`
+  - `gcc-12.2.0` -> `/cefs/SOME_INSTALL_ID/gcc-12.2.0`
+  - `gcc-trunk` -> `/cefs/DIFFERENT_ID/gcc-trunk`
+- Here we see the advantage of cefs: we minimize the number of actual squashfs images by relying on several installations being in the "same" squashfs image. We can consolidate images over time too.
+
+The expectation is there's one big squashfs install image holding all the stable compilers, regenerated monthly. And then several overlaying images, some for ad hoc builds of new compilers (e.g. new releases of GCC), and some for the daily builds.
+
+## Using
+
+To set up you'll need to run `sudo cefs install`. (it needs root to fiddle with autofs)
+
+After that you'll need to hack it so `/opt/compiler-explorer` is a symlink to an empty root image. I've actually locally linked `/opt/compiler-explorer` to `/home/mgodbolt/ce` in this instance so I can update it quickly as a mortal user without making `/opt` writable.
+
+To make an empty root like this:
+
+```
+$ sudo ln -sfT ~/ce /opt/compiler-explorer  # one-time setup only as root
+$ cefs create
+...lots of output...
+Fresh new cefs root created at /cefs/662a3bd6f57ac05b28032dbb5dd2bb489e9761c7
+$ ln -sfT /cefs/662a3bd6f57ac05b28032dbb5dd2bb489e9761c7 ~/ce
+$ ls /opt/compiler-explorer
+metadata.txt
+$ cat /opt/compiler-explorer/metadata.txt
+Initial empty image created at 2022-09-05 22:56:03.530590 by mgodbolt
+```
+
+Now you have an empty base layer.  After that you can layer on things using `ce_install buildroot` (terrible name needs changing):
+
+```
+$ ce_install buildroot gcc # literally all the GCCs...
+[...lots of work...]
+$ ls -l /opt/compiler-explorer/
+$ ls -l /opt/compiler-explorer/
+lrwxrwxrwx 1 root root  57 Sep  5 18:02 gcc-12.1.0 -> /cefs/fad719bc481ff3dbaffc7f5ef804c4b5ffc281d1/gcc-12.1.0/
+lrwxrwxrwx 1 root root  57 Sep  5 18:02 gcc-12.2.0 -> /cefs/fad719bc481ff3dbaffc7f5ef804c4b5ffc281d1/gcc-12.2.0/
+...etc
+```
+
+## TODO
+
+- cleanup
+- consolidation
+- everything, really
diff --git a/bin/lib/cefs.py b/bin/lib/cefs.py
index 4258e50ab..ad4160b82 100644
--- a/bin/lib/cefs.py
+++ b/bin/lib/cefs.py
@@ -1,10 +1,12 @@
+import getpass
 import logging
 import os
 import shutil
 import sys
 import subprocess
 from pathlib import Path
-from tempfile import mkdtemp, NamedTemporaryFile, TemporaryDirectory
+from tempfile import mkdtemp, TemporaryDirectory
+import datetime
 from typing import Optional, Mapping, List, Dict
 from dataclasses import dataclass
 
@@ -102,6 +104,18 @@ def install(context: CliContext):
     )
 
 
+@cli.command
+@click.pass_obj
+def create(context: CliContext):
+    """Create an empty image."""
+    creator = SquashFsCreator(squash_image_root=context.squash_image_root, cefs_root=context.cefs_root)
+    with creator as path:
+        image = CefsImage(cefs_root=context.cefs_root)
+        image.add_metadata(f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}")
+        image.render_to(path)
+    click.echo(f"Fresh new cefs root created at {creator.cefs_path}")
+
+
 class SquashFsCreator:
     def __init__(self, *, squash_image_root: Path, cefs_root: Path):
         self._squash_image_root = squash_image_root

From 636850210b9ab2d32b0720f641d6ac1d0862c0fd Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 6 Sep 2022 20:04:26 -0500
Subject: [PATCH 003/161] Some updates

---
 bin/lib/ce_install.py |  37 +++++++---------
 bin/lib/cefs.md       |   5 +--
 bin/lib/cefs.py       | 100 ++++++++++++++++++++++++++++++++++++++----
 bin/test/cefs_test.py |   6 +--
 4 files changed, 111 insertions(+), 37 deletions(-)

diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py
index f22a1046b..9884dd015 100644
--- a/bin/lib/ce_install.py
+++ b/bin/lib/ce_install.py
@@ -10,14 +10,13 @@
 from dataclasses import dataclass
 from functools import partial
 from pathlib import Path
-from tempfile import TemporaryDirectory
 from typing import List, Optional, Tuple
 
 import click
 import yaml
 
 from lib.amazon_properties import get_properties_compilers_and_libraries
-from lib.cefs import CefsImage, SquashFsCreator
+from lib.cefs import CefsImage, SquashFsCreator, CefsRoot
 from lib.config_safe_loader import ConfigSafeLoader
 from lib.installation import InstallationContext, installers_for, Installable
 from lib.library_yaml import LibraryYaml
@@ -371,22 +370,9 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
 
     installation_context = context.installation_context
 
-    if not installation_context.destination.is_symlink():
-        click.echo(f"Destination {installation_context.destination} is not a CEFS root symlink!")
-        sys.exit(1)
-    root_link_to_update = installation_context.destination
-    cefs_root_link = root_link_to_update.readlink()
-    while cefs_root_link.is_symlink():
-        root_link_to_update = cefs_root_link
-        cefs_root_link = cefs_root_link.readlink()
-    if not cefs_root_link.is_relative_to(cefs_root):
-        click.echo(
-            f"Destination {installation_context.destination} is not a CEFS root symlink "
-            f"({cefs_root_link} not relative to {cefs_root})!"
-        )
-        sys.exit(1)
-    current_image = CefsImage(cefs_root=cefs_root, directory=installation_context.destination)
-    current_image.add_metadata(f"Information read from root image {cefs_root_link}")
+    cefs = CefsRoot(fs_root=installation_context.destination, cefs_root=cefs_root)
+
+    current_image = cefs.read_image()
     for installable in context.get_installables(filter_):
         if force or installable.is_installed():
             dest_path = installation_context.destination / installable.install_path
@@ -399,10 +385,19 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
         installation_context._staging_root = tmp_path / "staging"
         installation_context.destination = tmp_path
         _LOGGER.info("Installing everything to a temp dir")
+        num_installed = 0
         for installable in context.get_installables(filter_):
             if Path(installable.install_path) not in current_image.catalog:
                 installable.install()
+                if not installable.is_installed():
+                    _LOGGER.error("%s installed OK, but doesn't appear as installed after", installable.name)
+                    sys.exit(1)
                 current_image.add_metadata(f"Installing {installable.install_path} from {installable}")
+                num_installed += 1
+
+    if not num_installed:
+        click.echo("No changes: not updating base image")
+        return
 
     new_squashfs_cefs = install_creator.cefs_path
     for installable in context.get_installables(filter_):
@@ -415,11 +410,9 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
         current_image.render_to(tmp_path)
 
     new_squashfs_cefs = root_creator.cefs_path
-    print(f"Output to {new_squashfs_cefs} replacing {cefs_root_link}")
 
-    # TODO keep track of previous? then we can "undo"
-    root_link_to_update.unlink(missing_ok=True)
-    root_link_to_update.symlink_to(new_squashfs_cefs)
+    click.echo(f"Built to {new_squashfs_cefs}")
+    cefs.update(new_squashfs_cefs)
 
 
 @cli.command()
diff --git a/bin/lib/cefs.md b/bin/lib/cefs.md
index d408fa160..d05936bd9 100644
--- a/bin/lib/cefs.md
+++ b/bin/lib/cefs.md
@@ -27,16 +27,15 @@ The expectation is there's one big squashfs install image holding all the stable
 
 To set up you'll need to run `sudo cefs install`. (it needs root to fiddle with autofs)
 
-After that you'll need to hack it so `/opt/compiler-explorer` is a symlink to an empty root image. I've actually locally linked `/opt/compiler-explorer` to `/home/mgodbolt/ce` in this instance so I can update it quickly as a mortal user without making `/opt` writable.
+I locally link `/opt/compiler-explorer` to `~/ce` in this instance so I can update it quickly as a mortal user without making `/opt` writable to my user.
 
 To make an empty root like this:
 
 ```
 $ sudo ln -sfT ~/ce /opt/compiler-explorer  # one-time setup only as root
-$ cefs create
+$ cefs create-root ~/ce
 ...lots of output...
 Fresh new cefs root created at /cefs/662a3bd6f57ac05b28032dbb5dd2bb489e9761c7
-$ ln -sfT /cefs/662a3bd6f57ac05b28032dbb5dd2bb489e9761c7 ~/ce
 $ ls /opt/compiler-explorer
 metadata.txt
 $ cat /opt/compiler-explorer/metadata.txt
diff --git a/bin/lib/cefs.py b/bin/lib/cefs.py
index ad4160b82..72adf97ea 100644
--- a/bin/lib/cefs.py
+++ b/bin/lib/cefs.py
@@ -1,14 +1,16 @@
+from __future__ import annotations
+
+import datetime
 import getpass
 import logging
 import os
 import shutil
-import sys
 import subprocess
+import sys
+from dataclasses import dataclass
 from pathlib import Path
 from tempfile import mkdtemp, TemporaryDirectory
-import datetime
 from typing import Optional, Mapping, List, Dict
-from dataclasses import dataclass
 
 import click
 
@@ -106,14 +108,28 @@ def install(context: CliContext):
 
 @cli.command
 @click.pass_obj
-def create(context: CliContext):
+def create_image(context: CliContext):
     """Create an empty image."""
+    created_path = _create_empty(context)
+    click.echo(f"Fresh new cefs root created at {created_path}")
+
+
+def _create_empty(context: CliContext) -> Path:
     creator = SquashFsCreator(squash_image_root=context.squash_image_root, cefs_root=context.cefs_root)
     with creator as path:
         image = CefsImage(cefs_root=context.cefs_root)
         image.add_metadata(f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}")
         image.render_to(path)
-    click.echo(f"Fresh new cefs root created at {creator.cefs_path}")
+    return creator.cefs_path
+
+
+@cli.command
+@click.argument("root", type=click.Path(file_okay=False, dir_okay=False, writable=True, path_type=Path), required=True)
+@click.pass_obj
+def create_root(context: CliContext, root: Path):
+    """Create an empty cefs root at ROOT."""
+    empty_path = _create_empty(context)
+    CefsRoot.create(base_image=empty_path, fs_root=root, cefs_root=context.cefs_root)
 
 
 class SquashFsCreator:
@@ -169,7 +185,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
             shutil.rmtree(self._path, ignore_errors=True)
 
 
-class BadCefsRoot(RuntimeError):
+class BadCefsImage(RuntimeError):
     pass
 
 
@@ -198,7 +214,7 @@ def _recurse_find_paths(self, path: Path, root_path: Path):
             if entry.is_symlink():
                 link = entry.readlink()
                 if not link.is_relative_to(self._cefs_root):
-                    raise BadCefsRoot(f"Found a symlink that wasn't a symlink to cefs: {entry} links to {link}")
+                    raise BadCefsImage(f"Found a symlink that wasn't a symlink to cefs: {entry} links to {link}")
                 _LOGGER.debug("Found existing %s -> %s", relative, link)
                 self._catalog[relative] = link
             elif entry.is_dir():
@@ -207,9 +223,9 @@ def _recurse_find_paths(self, path: Path, root_path: Path):
                 if relative == Path(METADATA_FILENAME):
                     self._metadata = entry.read_text(encoding="utf-8").splitlines(keepends=False)
                 else:
-                    raise BadCefsRoot(f"Found an unexpected file: {entry}")
+                    raise BadCefsImage(f"Found an unexpected file: {entry}")
             else:
-                raise BadCefsRoot(f"Found an unexpected entry: {entry}")
+                raise BadCefsImage(f"Found an unexpected entry: {entry}")
 
     def add_metadata(self, metadata: str) -> None:
         self._metadata.append(metadata)
@@ -236,6 +252,71 @@ def render_to(self, destination: Path) -> None:
             source_dir.symlink_to(dest)
 
 
+class BadCefsRoot(RuntimeError):
+    pass
+
+
+class CefsRoot:
+    """
+    Holds image about the root (ie a symlink) or a cefs.
+    TODO nomenclature is hard. image vs root?
+    """
+
+    def __init__(self, *, fs_root: Path, cefs_root: Path):
+        self._cefs_root = cefs_root
+        self._original_root = fs_root
+        self._root = fs_root
+        if not self._root.is_symlink():
+            raise BadCefsRoot(f"{self._root} is not a root - it's not a symlink")
+        root_link_to_update = self._root
+        self._image_root = root_link_to_update.readlink()
+        # Continue to follow along symlinks so the root can actually be a symlink to a symlink. We update the last
+        # symlink found along the path (to allow for `/some/root-owned` dir to symlink elsewhere: we actually update
+        # elsewhere.
+        while self._image_root.is_symlink():
+            _LOGGER.info(f"Following root symlink to {self._image_root}...")
+            self._root = self._image_root
+            self._image_root = self._root.readlink()
+
+        if not self._image_root.is_relative_to(cefs_root):
+            raise BadCefsRoot(
+                f"Destination {self._original_root} is not a CEFS root symlink "
+                f"({self._image_root} not relative to {self._cefs_root})!"
+            )
+
+    @classmethod
+    def create(cls, base_image: Path, fs_root: Path, cefs_root: Path) -> CefsRoot:
+        if fs_root.exists():
+            raise FileExistsError(f"{fs_root} already exists")
+        if not base_image.is_dir():
+            raise RuntimeError("Missing base image")
+        # Construct just to ensure it's a valid image.
+        CefsImage(cefs_root=cefs_root, directory=base_image)
+        fs_root.parent.mkdir(parents=True, exist_ok=True)
+        # TODO append a log?
+        fs_root.symlink_to(base_image, target_is_directory=True)
+        return cls(fs_root=fs_root, cefs_root=cefs_root)
+
+    @property
+    def fs_path(self) -> Path:
+        return self._root
+
+    @property
+    def image_root(self) -> Path:
+        return self._image_root
+
+    def read_image(self) -> CefsImage:
+        image = CefsImage(cefs_root=self._cefs_root, directory=self._image_root)
+        image.add_metadata(f"Information read from root image {self._image_root}")
+        return image
+
+    def update(self, new_path: Path) -> None:
+        # TODO append a log?
+        self._root.unlink(missing_ok=True)
+        self._root.symlink_to(new_path)
+        _LOGGER.info("Updated %s to %s", self._root, new_path)
+
+
 # TODO how to start the whole thing off? make an empty root and manually symlink it in position?
 # TODO cases where we have a GIANT image that only exists because one dir is in it
 #  ie repacking and refreshing old images
@@ -247,6 +328,7 @@ def render_to(self, destination: Path) -> None:
 # TODO detect cefs roots in `install` and redirect to buildroot?
 # TODO delete/etc
 # TODO handle old symlinks of trunk (and do trunk etc!)
+# TODO should "cefs_root" be onfigurable. better name?
 
 
 def main():
diff --git a/bin/test/cefs_test.py b/bin/test/cefs_test.py
index 96c6d2cf5..905e89414 100644
--- a/bin/test/cefs_test.py
+++ b/bin/test/cefs_test.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from lib.cefs import CefsImage, METADATA_FILENAME, BadCefsRoot
+from lib.cefs import CefsImage, METADATA_FILENAME, BadCefsImage
 
 
 @pytest.fixture
@@ -58,13 +58,13 @@ def test_cefs_can_read_existing_images_metadata(cefs_root: Path, image_root: Pat
 
 def test_cefs_is_upset_by_links_to_non_cefs(cefs_root: Path, image_root: Path, dest_root: Path):
     (image_root / "thing").symlink_to(dest_root)  # just a valid path not in cefs
-    with pytest.raises(BadCefsRoot, match="Found a symlink that wasn't a symlink to cefs"):
+    with pytest.raises(BadCefsImage, match="Found a symlink that wasn't a symlink to cefs"):
         CefsImage(directory=image_root, cefs_root=cefs_root)
 
 
 def test_cefs_is_upset_by_random_files(cefs_root: Path, image_root: Path, dest_root: Path):
     (image_root / "thing.txt").write_text("I am a mongoose", encoding="utf-8")
-    with pytest.raises(BadCefsRoot, match="Found an unexpected file"):
+    with pytest.raises(BadCefsImage, match="Found an unexpected file"):
         CefsImage(directory=image_root, cefs_root=cefs_root)
 
 

From 59f7794d4bfd368b79880902385208dff351ac59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micha=C5=82=20Cicho=C5=84?= <michcic@gmail.com>
Date: Tue, 6 Sep 2022 19:38:27 +0200
Subject: [PATCH 004/161] LLVM 15 (#799)

---
 bin/yaml/cpp.yaml       | 1 +
 bin/yaml/libraries.yaml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 5c38a6163..cbb76aa4f 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -561,6 +561,7 @@ compilers:
           - 13.0.0
           - 13.0.1
           - 14.0.0
+          - 15.0.0
     ellccs:
       check_exe: bin/clang++ --version
       dir: ellcc-{name}
diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index 427888ceb..9cebf7dc4 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -599,6 +599,7 @@ libraries:
       - 13.0.0
       - 13.0.1
       - 14.0.0
+      - 15.0.0
       type: s3tarballs
     magic_enum:
       check_file: include/magic_enum.hpp

From 9dc26f642386032c302cc6c5d8c4bcf684f1980b Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 08:13:28 -0500
Subject: [PATCH 005/161] Refactor and splitting. Renames

---
 bin/cefs                       |   2 +-
 bin/lib/ce_install.py          |  24 +--
 bin/lib/cefs.py                | 339 ---------------------------------
 bin/lib/cefs/__init__.py       |   0
 bin/lib/cefs/__main__.py       | 155 +++++++++++++++
 bin/lib/cefs/config.py         |  10 +
 bin/lib/cefs/root.py           |  73 +++++++
 bin/lib/cefs/root_image.py     |  75 ++++++++
 bin/lib/cefs/squash.py         |  65 +++++++
 bin/scripts/python_module_shim |  21 ++
 bin/test/cefs_test.py          |  70 +++----
 11 files changed, 448 insertions(+), 386 deletions(-)
 delete mode 100644 bin/lib/cefs.py
 create mode 100644 bin/lib/cefs/__init__.py
 create mode 100644 bin/lib/cefs/__main__.py
 create mode 100644 bin/lib/cefs/config.py
 create mode 100644 bin/lib/cefs/root.py
 create mode 100644 bin/lib/cefs/root_image.py
 create mode 100644 bin/lib/cefs/squash.py
 create mode 100755 bin/scripts/python_module_shim

diff --git a/bin/cefs b/bin/cefs
index ae249b7dd..49cb85b03 120000
--- a/bin/cefs
+++ b/bin/cefs
@@ -1 +1 @@
-scripts/python_shim
\ No newline at end of file
+scripts/python_module_shim
\ No newline at end of file
diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py
index 9884dd015..fdb55ec94 100644
--- a/bin/lib/ce_install.py
+++ b/bin/lib/ce_install.py
@@ -16,7 +16,9 @@
 import yaml
 
 from lib.amazon_properties import get_properties_compilers_and_libraries
-from lib.cefs import CefsImage, SquashFsCreator, CefsRoot
+from lib.cefs.config import CefsConfig
+from lib.cefs.root import CefsFsRoot
+from lib.cefs.squash import SquashFsCreator
 from lib.config_safe_loader import ConfigSafeLoader
 from lib.installation import InstallationContext, installers_for, Installable
 from lib.library_yaml import LibraryYaml
@@ -347,11 +349,11 @@ def squash(context: CliContext, filter_: List[str], force: bool, image_dir: Path
 @click.pass_obj
 @click.option("--force", is_flag=True, help="Force even if would otherwise skip")
 @click.option(
-    "--cefs-root",
+    "--cefs-mountpoint",
     default=Path("/cefs"),
-    metavar="CEFS_ROOT",
+    metavar="MOUNTPOINT",
     type=click.Path(file_okay=False, path_type=Path),
-    help="Install or assume cefs is installed at CEFS_ROOT",
+    help="Install or assume cefs is to use MOUNTPOINT",
     show_default=True,
 )
 @click.option(
@@ -363,16 +365,16 @@ def squash(context: CliContext, filter_: List[str], force: bool, image_dir: Path
     show_default=True,
 )
 @click.argument("filter_", metavar="FILTER", nargs=-1)
-def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image_root: Path, cefs_root: Path):
+def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image_root: Path, cefs_mountpoint: Path):
     """Squash all things matching to a single root image."""
 
     # check for things already installed in the _current_ root?
 
     installation_context = context.installation_context
+    cefs_config = CefsConfig(mountpoint=cefs_mountpoint, image_root=squash_image_root)
+    fs_root = CefsFsRoot(fs_root=installation_context.destination, config=cefs_config)
 
-    cefs = CefsRoot(fs_root=installation_context.destination, cefs_root=cefs_root)
-
-    current_image = cefs.read_image()
+    current_image = fs_root.read_image()
     for installable in context.get_installables(filter_):
         if force or installable.is_installed():
             dest_path = installation_context.destination / installable.install_path
@@ -380,7 +382,7 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
                 _LOGGER.error("Found an installable that wasn't a symlink: %s", dest_path)
                 sys.exit(1)
 
-    install_creator = SquashFsCreator(squash_image_root=squash_image_root, cefs_root=cefs_root)
+    install_creator = SquashFsCreator(config=cefs_config)
     with install_creator as tmp_path:
         installation_context._staging_root = tmp_path / "staging"
         installation_context.destination = tmp_path
@@ -405,14 +407,14 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
             current_image.link_path(Path(installable.install_path), new_squashfs_cefs / installable.install_path)
 
     _LOGGER.info("Building new root fs")
-    root_creator = SquashFsCreator(squash_image_root=squash_image_root, cefs_root=cefs_root)
+    root_creator = SquashFsCreator(config=cefs_config)
     with root_creator as tmp_path:
         current_image.render_to(tmp_path)
 
     new_squashfs_cefs = root_creator.cefs_path
 
     click.echo(f"Built to {new_squashfs_cefs}")
-    cefs.update(new_squashfs_cefs)
+    fs_root.update(new_squashfs_cefs)
 
 
 @cli.command()
diff --git a/bin/lib/cefs.py b/bin/lib/cefs.py
deleted file mode 100644
index 72adf97ea..000000000
--- a/bin/lib/cefs.py
+++ /dev/null
@@ -1,339 +0,0 @@
-from __future__ import annotations
-
-import datetime
-import getpass
-import logging
-import os
-import shutil
-import subprocess
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-from tempfile import mkdtemp, TemporaryDirectory
-from typing import Optional, Mapping, List, Dict
-
-import click
-
-_LOGGER = logging.getLogger(__name__)
-METADATA_FILENAME = "metadata.txt"
-
-
-@dataclass(frozen=True)
-class CliContext:
-    cefs_root: Path
-    squash_image_root: Path
-
-
-@click.group()
-@click.option(
-    "--cefs-root",
-    default=Path("/cefs"),
-    metavar="CEFS_ROOT",
-    type=click.Path(file_okay=False, path_type=Path),
-    help="Install or assume cefs is installed at CEFS_ROOT",
-    show_default=True,
-)
-@click.option(
-    "--squash-image-root",
-    default=Path("/opt/cefs-images"),
-    metavar="IMAGE_DIR",
-    type=click.Path(file_okay=False, path_type=Path),
-    help="Store or look for squashfs images in IMAGE_DIR",
-    show_default=True,
-)
-@click.option("--debug/--no-debug", help="Turn on debugging")
-@click.option("--log-to-console", is_flag=True, help="Log output to console, even if logging to a file is requested")
-@click.option("--log", metavar="LOGFILE", help="Log to LOGFILE", type=click.Path(dir_okay=False, writable=True))
-@click.pass_context
-def cli(
-    ctx: click.Context,
-    cefs_root: Path,
-    squash_image_root: Path,
-    debug: bool,
-    log_to_console: bool,
-    log: Optional[str],
-):
-    """Administrate the Compiler Explorer File System (cefs)."""
-    formatter = logging.Formatter(fmt="%(asctime)s %(name)-15s %(levelname)-8s %(message)s")
-    root_logger = logging.getLogger()
-    root_logger.setLevel(logging.DEBUG if debug else logging.INFO)
-    if log:
-        file_handler = logging.FileHandler(log)
-        file_handler.setFormatter(formatter)
-        root_logger.addHandler(file_handler)
-    if not log or log_to_console:
-        console_handler = logging.StreamHandler(sys.stdout)
-        console_handler.setFormatter(formatter)
-        root_logger.addHandler(console_handler)
-    ctx.obj = CliContext(cefs_root=cefs_root, squash_image_root=squash_image_root)
-
-
-@cli.command
-@click.pass_obj
-def install(context: CliContext):
-    """Install the CEFS. Needs to be run as root."""
-    if os.geteuid() != 0:
-        click.echo("Installing cefs needs root privileges.")
-        sys.exit(1)
-    if not Path("/etc/auto.master.d").is_dir():
-        click.echo("Installing cefs needs autofs. Please install manually using e.g. `apt install autofs`")
-        sys.exit(1)
-    click.echo("Writing config files...")
-    auto_cefs_config_file = Path("/etc/auto.cefs")
-    auto_cefs_config_file.write_text(
-        f"* -fstype=squashfs,loop,nosuid,nodev,ro :{context.squash_image_root}/&.sqfs\n", encoding="utf-8"
-    )
-    Path("/etc/auto.master.d/cefs.autofs").write_text(
-        f"{context.cefs_root} {auto_cefs_config_file}  --negative-timeout 1", encoding="utf-8"
-    )
-    if not context.squash_image_root.is_dir():
-        click.echo(
-            f"Creating {context.squash_image_root} squash image root.\n"
-            f"By default this will be writable BY ALL USERS, but is sticky like /tmp. This is to make it usable\n"
-            f"by unprivileged users. You are responsible for making sure that's ok, and changing the directory\n"
-            f"permissions as appropriate."
-        )
-        context.squash_image_root.mkdir(parents=True)
-        context.squash_image_root.chmod(0o1777)
-    click.echo("Restarting autofs")
-    subprocess.check_call(["service", "autofs", "restart"])
-    click.echo(
-        f"CEFS installed with a root at {context.cefs_root} and squash image root at {context.squash_image_root}."
-    )
-    click.echo(
-        f"Note that the root {context.cefs_root} will appear empty, "
-        f"but will automatically mount on demand when required."
-    )
-
-
-@cli.command
-@click.pass_obj
-def create_image(context: CliContext):
-    """Create an empty image."""
-    created_path = _create_empty(context)
-    click.echo(f"Fresh new cefs root created at {created_path}")
-
-
-def _create_empty(context: CliContext) -> Path:
-    creator = SquashFsCreator(squash_image_root=context.squash_image_root, cefs_root=context.cefs_root)
-    with creator as path:
-        image = CefsImage(cefs_root=context.cefs_root)
-        image.add_metadata(f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}")
-        image.render_to(path)
-    return creator.cefs_path
-
-
-@cli.command
-@click.argument("root", type=click.Path(file_okay=False, dir_okay=False, writable=True, path_type=Path), required=True)
-@click.pass_obj
-def create_root(context: CliContext, root: Path):
-    """Create an empty cefs root at ROOT."""
-    empty_path = _create_empty(context)
-    CefsRoot.create(base_image=empty_path, fs_root=root, cefs_root=context.cefs_root)
-
-
-class SquashFsCreator:
-    def __init__(self, *, squash_image_root: Path, cefs_root: Path):
-        self._squash_image_root = squash_image_root
-        self._cefs_root = cefs_root
-        self._sha = None
-
-    def _close(self):
-        with TemporaryDirectory(prefix="ce-squash-builder") as tmp_dir:
-            tmp_sqfs = Path(tmp_dir) / "temp.sqfs"
-            subprocess.check_call(
-                [
-                    "/usr/bin/mksquashfs",
-                    str(self._path),
-                    str(tmp_sqfs),
-                    "-all-root",
-                    "-root-mode",
-                    "755",
-                    "-progress",
-                    "-comp",
-                    "zstd",
-                ]
-            )
-            self._sha, _filename = subprocess.check_output(["/usr/bin/shasum", str(tmp_sqfs)]).decode("utf-8").split()
-            if not self.image.exists():
-                _LOGGER.info("New squashfs image: %s", self.image)
-                tmp_sqfs.replace(self.image)
-            else:
-                _LOGGER.info("Existing: %s", self.image)
-                tmp_sqfs.unlink()
-
-    @property
-    def image(self) -> Path:
-        assert self._sha is not None
-        return self._squash_image_root / f"{self._sha}.sqfs"
-
-    @property
-    def cefs_path(self) -> Path:
-        assert self._sha is not None
-        return self._cefs_root / self._sha
-
-    def __enter__(self):
-        self._path = Path(mkdtemp(prefix="ce-install-temp"))
-        return self._path
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        try:
-            if not exc_type:
-                self._close()
-        finally:
-            subprocess.check_call(["chmod", "-R", "u+w", str(self._path)])
-            shutil.rmtree(self._path, ignore_errors=True)
-
-
-class BadCefsImage(RuntimeError):
-    pass
-
-
-class BadCefsLink(RuntimeError):
-    pass
-
-
-class CefsImage:
-    """
-    Holds information about a CEFS root image, which is a directory full of symlinks to
-    other CEFS images, and some metadata.
-    """
-
-    def __init__(self, *, cefs_root: Path, directory: Optional[Path] = None):
-        self._cefs_root = cefs_root.resolve(strict=True)
-        self._catalog: Dict[Path, Path] = {}
-        self._metadata: List[str] = []
-
-        if directory:
-            self._recurse_find_paths(directory, directory)
-            _LOGGER.info("CEFS image at %s has %d entries", directory, len(self.catalog))
-
-    def _recurse_find_paths(self, path: Path, root_path: Path):
-        for entry in path.iterdir():
-            relative = entry.relative_to(root_path)
-            if entry.is_symlink():
-                link = entry.readlink()
-                if not link.is_relative_to(self._cefs_root):
-                    raise BadCefsImage(f"Found a symlink that wasn't a symlink to cefs: {entry} links to {link}")
-                _LOGGER.debug("Found existing %s -> %s", relative, link)
-                self._catalog[relative] = link
-            elif entry.is_dir():
-                self._recurse_find_paths(entry, root_path)
-            elif entry.is_file():
-                if relative == Path(METADATA_FILENAME):
-                    self._metadata = entry.read_text(encoding="utf-8").splitlines(keepends=False)
-                else:
-                    raise BadCefsImage(f"Found an unexpected file: {entry}")
-            else:
-                raise BadCefsImage(f"Found an unexpected entry: {entry}")
-
-    def add_metadata(self, metadata: str) -> None:
-        self._metadata.append(metadata)
-
-    def link_path(self, subdir: Path, cefs_link: Path):
-        if not cefs_link.is_relative_to(self._cefs_root):
-            raise BadCefsLink(f"Link is not relative to cefs: {cefs_link}")
-        self._catalog[subdir] = cefs_link
-
-    @property
-    def catalog(self) -> Mapping[Path, Path]:
-        return self._catalog
-
-    @property
-    def metadata(self) -> List[str]:
-        return self._metadata
-
-    def render_to(self, destination: Path) -> None:
-        destination.mkdir(parents=True, exist_ok=True)
-        (destination / METADATA_FILENAME).write_text("\n".join(self._metadata) + "\n", encoding="utf-8")
-        for entry, dest in self.catalog.items():
-            source_dir = destination / entry
-            source_dir.parent.mkdir(parents=True, exist_ok=True)
-            source_dir.symlink_to(dest)
-
-
-class BadCefsRoot(RuntimeError):
-    pass
-
-
-class CefsRoot:
-    """
-    Holds image about the root (ie a symlink) or a cefs.
-    TODO nomenclature is hard. image vs root?
-    """
-
-    def __init__(self, *, fs_root: Path, cefs_root: Path):
-        self._cefs_root = cefs_root
-        self._original_root = fs_root
-        self._root = fs_root
-        if not self._root.is_symlink():
-            raise BadCefsRoot(f"{self._root} is not a root - it's not a symlink")
-        root_link_to_update = self._root
-        self._image_root = root_link_to_update.readlink()
-        # Continue to follow along symlinks so the root can actually be a symlink to a symlink. We update the last
-        # symlink found along the path (to allow for `/some/root-owned` dir to symlink elsewhere: we actually update
-        # elsewhere.
-        while self._image_root.is_symlink():
-            _LOGGER.info(f"Following root symlink to {self._image_root}...")
-            self._root = self._image_root
-            self._image_root = self._root.readlink()
-
-        if not self._image_root.is_relative_to(cefs_root):
-            raise BadCefsRoot(
-                f"Destination {self._original_root} is not a CEFS root symlink "
-                f"({self._image_root} not relative to {self._cefs_root})!"
-            )
-
-    @classmethod
-    def create(cls, base_image: Path, fs_root: Path, cefs_root: Path) -> CefsRoot:
-        if fs_root.exists():
-            raise FileExistsError(f"{fs_root} already exists")
-        if not base_image.is_dir():
-            raise RuntimeError("Missing base image")
-        # Construct just to ensure it's a valid image.
-        CefsImage(cefs_root=cefs_root, directory=base_image)
-        fs_root.parent.mkdir(parents=True, exist_ok=True)
-        # TODO append a log?
-        fs_root.symlink_to(base_image, target_is_directory=True)
-        return cls(fs_root=fs_root, cefs_root=cefs_root)
-
-    @property
-    def fs_path(self) -> Path:
-        return self._root
-
-    @property
-    def image_root(self) -> Path:
-        return self._image_root
-
-    def read_image(self) -> CefsImage:
-        image = CefsImage(cefs_root=self._cefs_root, directory=self._image_root)
-        image.add_metadata(f"Information read from root image {self._image_root}")
-        return image
-
-    def update(self, new_path: Path) -> None:
-        # TODO append a log?
-        self._root.unlink(missing_ok=True)
-        self._root.symlink_to(new_path)
-        _LOGGER.info("Updated %s to %s", self._root, new_path)
-
-
-# TODO how to start the whole thing off? make an empty root and manually symlink it in position?
-# TODO cases where we have a GIANT image that only exists because one dir is in it
-#  ie repacking and refreshing old images
-# TODO consolidation in general! aim is to have very few layers, after all.
-# TODO delete old things
-# TODO keep track of a symlink and its history /opt/compiler-explorer --> /opt/compiler-explorer.history?
-# TODO better naming of dirs/manifest for each so we know who and why created each layer?
-# TODO should we separate out root squashfs images from data?
-# TODO detect cefs roots in `install` and redirect to buildroot?
-# TODO delete/etc
-# TODO handle old symlinks of trunk (and do trunk etc!)
-# TODO should "cefs_root" be onfigurable. better name?
-
-
-def main():
-    cli(prog_name="cefs")  # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
-
-
-if __name__ == "__main__":
-    main()
diff --git a/bin/lib/cefs/__init__.py b/bin/lib/cefs/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
new file mode 100644
index 000000000..3f41daa48
--- /dev/null
+++ b/bin/lib/cefs/__main__.py
@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+import datetime
+import getpass
+import logging
+import os
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+import click
+
+from lib.cefs.config import CefsConfig
+from lib.cefs.root import CefsFsRoot
+from lib.cefs.root_image import CefsRootImage
+from lib.cefs.squash import SquashFsCreator
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class CliContext:
+    config: CefsConfig
+
+
+@click.group()
+@click.option(
+    "--cefs-mountpoint",
+    default=Path("/cefs"),
+    metavar="MOUNTPOINT",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Install or assume cefs is to use MOUNTPOINT",
+    show_default=True,
+)
+@click.option(
+    "--squash-image-root",
+    default=Path("/opt/cefs-images"),
+    metavar="IMAGE_DIR",
+    type=click.Path(file_okay=False, path_type=Path),
+    help="Store or look for squashfs images in IMAGE_DIR",
+    show_default=True,
+)
+@click.option("--debug/--no-debug", help="Turn on debugging")
+@click.option("--log-to-console", is_flag=True, help="Log output to console, even if logging to a file is requested")
+@click.option("--log", metavar="LOGFILE", help="Log to LOGFILE", type=click.Path(dir_okay=False, writable=True))
+@click.pass_context
+def cli(
+    ctx: click.Context,
+    cefs_mountpoint: Path,
+    squash_image_root: Path,
+    debug: bool,
+    log_to_console: bool,
+    log: Optional[str],
+):
+    """Administrate the Compiler Explorer File System (cefs)."""
+    formatter = logging.Formatter(fmt="%(asctime)s %(name)-15s %(levelname)-8s %(message)s")
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG if debug else logging.INFO)
+    if log:
+        file_handler = logging.FileHandler(log)
+        file_handler.setFormatter(formatter)
+        root_logger.addHandler(file_handler)
+    if not log or log_to_console:
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setFormatter(formatter)
+        root_logger.addHandler(console_handler)
+    ctx.obj = CliContext(config=CefsConfig(mountpoint=cefs_mountpoint, image_root=squash_image_root))
+
+
+@cli.command
+@click.pass_obj
+def install(context: CliContext):
+    """Install the CEFS mountpoint and associated directories. Needs to be run as root."""
+    config = context.config
+    if os.geteuid() != 0:
+        click.echo("Installing cefs needs root privileges.")
+        sys.exit(1)
+    if not Path("/etc/auto.master.d").is_dir():
+        click.echo("Installing cefs needs autofs. Please install manually using e.g. `apt install autofs`")
+        sys.exit(1)
+    click.echo("Writing config files...")
+    auto_cefs_config_file = Path("/etc/auto.cefs")
+    auto_cefs_config_file.write_text(
+        f"* -fstype=squashfs,loop,nosuid,nodev,ro :{config.image_root}/&.sqfs\n", encoding="utf-8"
+    )
+    Path("/etc/auto.master.d/cefs.autofs").write_text(
+        f"{config.mountpoint} {auto_cefs_config_file}  --negative-timeout 1", encoding="utf-8"
+    )
+    if not config.image_root.is_dir():
+        click.echo(
+            f"Creating {config.image_root} squash image root.\n"
+            f"By default this will be writable BY ALL USERS, but is sticky like /tmp. This is to make it usable\n"
+            f"by unprivileged users. You are responsible for making sure that's ok, and changing the directory\n"
+            f"permissions as appropriate."
+        )
+        config.image_root.mkdir(parents=True)
+        config.image_root.chmod(0o1777)
+    click.echo("Restarting autofs")
+    subprocess.check_call(["service", "autofs", "restart"])
+    click.echo(f"CEFS installed with a root at {config.mountpoint} and squash image root at {config.image_root}.")
+    click.echo(
+        f"Note that the root {config.mountpoint} will appear empty, "
+        f"but will automatically mount on demand when required."
+    )
+
+
+@cli.command
+@click.pass_obj
+def create_image(context: CliContext):
+    """Create an empty image."""
+    created_path = _create_empty(context.config)
+    click.echo(f"Fresh new cefs image created at {created_path}")
+
+
+def _create_empty(config: CefsConfig) -> Path:
+    creator = SquashFsCreator(config)
+    with creator as path:
+        image = CefsRootImage(cefs_mountpoint=config.mountpoint)
+        image.add_metadata(f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}")
+        image.render_to(path)
+    return creator.cefs_path
+
+
+@cli.command
+@click.argument("root", type=click.Path(file_okay=False, dir_okay=False, writable=True, path_type=Path), required=True)
+@click.pass_obj
+def create_root(context: CliContext, root: Path):
+    """Create an empty cefs root at ROOT."""
+    empty_path = _create_empty(context.config)
+    CefsFsRoot.create(base_image=empty_path, fs_root=root, config=context.config)
+
+
+# TODO how to start the whole thing off? make an empty root and manually symlink it in position?
+# TODO cases where we have a GIANT image that only exists because one dir is in it
+#  ie repacking and refreshing old images
+# TODO consolidation in general! aim is to have very few layers, after all.
+# TODO delete old things
+# TODO keep track of a symlink and its history /opt/compiler-explorer --> /opt/compiler-explorer.history?
+# TODO better naming of dirs/manifest for each so we know who and why created each layer?
+# TODO should we separate out root squashfs images from data?
+# TODO detect cefs roots in `install` and redirect to buildroot?
+# TODO delete/etc
+# TODO handle old symlinks of trunk (and do trunk etc!)
+# TODO should "cefs_mountpoint" be onfigurable.
+# TODO should there be a magic file? should metadata.txt by .metadata.txt or indeed just that special file?
+
+
+def main():
+    cli(prog_name="cefs")  # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bin/lib/cefs/config.py b/bin/lib/cefs/config.py
new file mode 100644
index 000000000..0082701d6
--- /dev/null
+++ b/bin/lib/cefs/config.py
@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class CefsConfig:
+    mountpoint: Path
+    image_root: Path
diff --git a/bin/lib/cefs/root.py b/bin/lib/cefs/root.py
new file mode 100644
index 000000000..5a9bc338f
--- /dev/null
+++ b/bin/lib/cefs/root.py
@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from lib.cefs.config import CefsConfig
+from lib.cefs.root_image import CefsRootImage
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class BadCefsRoot(RuntimeError):
+    pass
+
+
+class CefsFsRoot:
+    """
+    Holds image about a cefs filesystem root: a symlink pointing at a CefsRootImage on a cefs mountpoint.
+    """
+
+    def __init__(self, *, fs_root: Path, config: CefsConfig):
+        self._config = config
+        self._original_root = fs_root
+        self._root = fs_root
+        if not self._root.is_symlink():
+            raise BadCefsRoot(f"{self._root} is not a cefs filesystem root - it's not a symlink")
+        root_link_to_update = self._root
+        self._image_root = root_link_to_update.readlink()
+        # Continue to follow along symlinks so the root can actually be a symlink to a symlink. We update the last
+        # symlink found along the path (to allow for `/some/root-owned` dir to symlink elsewhere: we actually update
+        # elsewhere.
+        while self._image_root.is_symlink():
+            _LOGGER.info(f"Following root symlink to {self._image_root}...")
+            self._root = self._image_root
+            self._image_root = self._root.readlink()
+
+        if not self._image_root.is_relative_to(self._config.mountpoint):
+            raise BadCefsRoot(
+                f"Destination {self._original_root} is not a CEFS root symlink "
+                f"({self._image_root} not relative to {self._config.mountpoint})!"
+            )
+
+    @classmethod
+    def create(cls, base_image: Path, fs_root: Path, config: CefsConfig) -> CefsFsRoot:
+        if fs_root.exists():
+            raise FileExistsError(f"{fs_root} already exists")
+        if not base_image.is_dir():
+            raise RuntimeError("Missing base image")
+        # Construct just to ensure it's a valid image.
+        CefsRootImage(cefs_mountpoint=config.mountpoint, directory=base_image)
+        fs_root.parent.mkdir(parents=True, exist_ok=True)
+        # TODO append a log?
+        fs_root.symlink_to(base_image, target_is_directory=True)
+        return cls(fs_root=fs_root, config=config)
+
+    @property
+    def fs_path(self) -> Path:
+        return self._root
+
+    @property
+    def image_root(self) -> Path:
+        return self._image_root
+
+    def read_image(self) -> CefsRootImage:
+        image = CefsRootImage(cefs_mountpoint=self._config.mountpoint, directory=self._image_root)
+        image.add_metadata(f"Information read from root image {self._image_root}")
+        return image
+
+    def update(self, new_path: Path) -> None:
+        # TODO append a log?
+        self._root.unlink(missing_ok=True)
+        self._root.symlink_to(new_path)
+        _LOGGER.info("Updated %s to %s", self._root, new_path)
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
new file mode 100644
index 000000000..c4766e645
--- /dev/null
+++ b/bin/lib/cefs/root_image.py
@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Optional, Dict, List, Mapping
+
+_LOGGER = logging.getLogger(__name__)
+METADATA_FILENAME = "metadata.txt"
+
+
+class BadCefsImage(RuntimeError):
+    pass
+
+
+class BadCefsLink(RuntimeError):
+    pass
+
+
+class CefsRootImage:
+    """
+    Holds information about a CEFS root image, which is a directory full of symlinks to other CEFS images, and some
+    metadata. TODO naming
+    """
+
+    def __init__(self, *, cefs_mountpoint: Path, directory: Optional[Path] = None):
+        self._cefs_mountpoint = cefs_mountpoint.resolve(strict=True)
+        self._catalog: Dict[Path, Path] = {}
+        self._metadata: List[str] = []
+
+        if directory:
+            self._recurse_find_paths(directory, directory)
+            _LOGGER.info("CEFS image at %s has %d entries", directory, len(self.catalog))
+
+    def _recurse_find_paths(self, path: Path, root_path: Path):
+        for entry in path.iterdir():
+            relative = entry.relative_to(root_path)
+            if entry.is_symlink():
+                link = entry.readlink()
+                if not link.is_relative_to(self._cefs_mountpoint):
+                    raise BadCefsImage(f"Found a symlink that wasn't a symlink to cefs: {entry} links to {link}")
+                _LOGGER.debug("Found existing %s -> %s", relative, link)
+                self._catalog[relative] = link
+            elif entry.is_dir():
+                self._recurse_find_paths(entry, root_path)
+            elif entry.is_file():
+                if relative == Path(METADATA_FILENAME):
+                    self._metadata = entry.read_text(encoding="utf-8").splitlines(keepends=False)
+                else:
+                    raise BadCefsImage(f"Found an unexpected file: {entry}")
+            else:
+                raise BadCefsImage(f"Found an unexpected entry: {entry}")
+
+    def add_metadata(self, metadata: str) -> None:
+        self._metadata.append(metadata)
+
+    def link_path(self, subdir: Path, cefs_link: Path):
+        if not cefs_link.is_relative_to(self._cefs_mountpoint):
+            raise BadCefsLink(f"Link is not relative to cefs: {cefs_link}")
+        self._catalog[subdir] = cefs_link
+
+    @property
+    def catalog(self) -> Mapping[Path, Path]:
+        return self._catalog
+
+    @property
+    def metadata(self) -> List[str]:
+        return self._metadata
+
+    def render_to(self, destination: Path) -> None:
+        destination.mkdir(parents=True, exist_ok=True)
+        (destination / METADATA_FILENAME).write_text("\n".join(self._metadata) + "\n", encoding="utf-8")
+        for entry, dest in self.catalog.items():
+            source_dir = destination / entry
+            source_dir.parent.mkdir(parents=True, exist_ok=True)
+            source_dir.symlink_to(dest)
diff --git a/bin/lib/cefs/squash.py b/bin/lib/cefs/squash.py
new file mode 100644
index 000000000..ec15e0a08
--- /dev/null
+++ b/bin/lib/cefs/squash.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+from tempfile import TemporaryDirectory, mkdtemp
+
+from lib.cefs.config import CefsConfig
+
+
+import logging
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class SquashFsCreator:
+    def __init__(self, config: CefsConfig):
+        self._config = config
+        self._sha = None
+
+    def _close(self):
+        with TemporaryDirectory(prefix="ce-squash-builder") as tmp_dir:
+            tmp_sqfs = Path(tmp_dir) / "temp.sqfs"
+            subprocess.check_call(
+                [
+                    "/usr/bin/mksquashfs",
+                    str(self._path),
+                    str(tmp_sqfs),
+                    "-all-root",
+                    "-root-mode",
+                    "755",
+                    "-progress",
+                    "-comp",
+                    "zstd",
+                ]
+            )
+            self._sha, _filename = subprocess.check_output(["/usr/bin/shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            if not self.image.exists():
+                _LOGGER.info("New squashfs image: %s", self.image)
+                tmp_sqfs.replace(self.image)
+            else:
+                _LOGGER.info("Existing: %s", self.image)
+                tmp_sqfs.unlink()
+
+    @property
+    def image(self) -> Path:
+        assert self._sha is not None
+        return self._config.image_root / f"{self._sha}.sqfs"
+
+    @property
+    def cefs_path(self) -> Path:
+        assert self._sha is not None
+        return self._config.mountpoint / self._sha
+
+    def __enter__(self):
+        self._path = Path(mkdtemp(prefix="ce-install-temp"))
+        return self._path
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        try:
+            if not exc_type:
+                self._close()
+        finally:
+            subprocess.check_call(["chmod", "-R", "u+w", str(self._path)])
+            shutil.rmtree(self._path, ignore_errors=True)
diff --git a/bin/scripts/python_module_shim b/bin/scripts/python_module_shim
new file mode 100755
index 000000000..ddedf9fee
--- /dev/null
+++ b/bin/scripts/python_module_shim
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ "$(uname -s)" = "Darwin" ]]; then
+  ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+else
+  ROOT_DIR=$(readlink -f "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/../..")
+fi
+MY_NAME=$(basename "$0")
+echo ${ROOT_DIR}
+
+if ! make -sq -C "${ROOT_DIR}" ce; then
+    echo "python environment is out of date"
+    echo "please run 'make ce' in the top level directory"
+    echo "(this is not done automatically as it can be a lengthy and surprising process)"
+    exit 1
+fi
+
+export PYTHONPATH=${ROOT_DIR}/bin
+exec "${ROOT_DIR}/.env/bin/python3" -m "lib.${MY_NAME}" "$@"
diff --git a/bin/test/cefs_test.py b/bin/test/cefs_test.py
index 905e89414..7c750e122 100644
--- a/bin/test/cefs_test.py
+++ b/bin/test/cefs_test.py
@@ -2,11 +2,11 @@
 
 import pytest
 
-from lib.cefs import CefsImage, METADATA_FILENAME, BadCefsImage
+from lib.cefs.root_image import CefsRootImage, METADATA_FILENAME, BadCefsImage
 
 
 @pytest.fixture
-def cefs_root(tmp_path) -> Path:
+def cefs_mountpoint(tmp_path) -> Path:
     result = tmp_path / "some-cefs-root"
     result.mkdir()
     return result
@@ -26,56 +26,56 @@ def dest_root(tmp_path) -> Path:
     return result
 
 
-def test_cefs_root_should_handle_no_initial_path(cefs_root: Path):
-    root = CefsImage(cefs_root=cefs_root)
+def test_cefs_root_should_handle_no_initial_path(cefs_mountpoint):
+    root = CefsRootImage(cefs_mountpoint=cefs_mountpoint)
     assert not root.catalog
     assert not root.metadata
 
 
-def test_cefs_root_should_handle_empty_dirs(cefs_root: Path, image_root: Path):
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+def test_cefs_root_should_handle_empty_dirs(cefs_mountpoint, image_root: Path):
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
     assert not root.catalog
     assert not root.metadata
 
 
-def test_cefs_can_read_existing_images(cefs_root: Path, image_root: Path, dest_root: Path):
-    (image_root / "thing").symlink_to(cefs_root / "thing")
+def test_cefs_can_read_existing_images(cefs_mountpoint, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_mountpoint / "thing")
     (image_root / "some" / "sub").mkdir(parents=True)
-    (image_root / "some" / "sub" / "directory").symlink_to(cefs_root / "thing" / "in" / "subdir")
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    (image_root / "some" / "sub" / "directory").symlink_to(cefs_mountpoint / "thing" / "in" / "subdir")
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
     assert root.catalog == {
-        Path("thing"): cefs_root / "thing",
-        Path("some") / "sub" / "directory": cefs_root / "thing" / "in" / "subdir",
+        Path("thing"): cefs_mountpoint / "thing",
+        Path("some") / "sub" / "directory": cefs_mountpoint / "thing" / "in" / "subdir",
     }
     assert not root.metadata
 
 
-def test_cefs_can_read_existing_images_metadata(cefs_root: Path, image_root: Path, dest_root: Path):
+def test_cefs_can_read_existing_images_metadata(cefs_mountpoint, image_root: Path, dest_root: Path):
     (image_root / METADATA_FILENAME).write_text("Meta\nData\nFTW\n", encoding="utf-8")
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
     assert root.metadata == ["Meta", "Data", "FTW"]
 
 
-def test_cefs_is_upset_by_links_to_non_cefs(cefs_root: Path, image_root: Path, dest_root: Path):
+def test_cefs_is_upset_by_links_to_non_cefs(cefs_mountpoint, image_root: Path, dest_root: Path):
     (image_root / "thing").symlink_to(dest_root)  # just a valid path not in cefs
     with pytest.raises(BadCefsImage, match="Found a symlink that wasn't a symlink to cefs"):
-        CefsImage(directory=image_root, cefs_root=cefs_root)
+        CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
 
 
-def test_cefs_is_upset_by_random_files(cefs_root: Path, image_root: Path, dest_root: Path):
+def test_cefs_is_upset_by_random_files(cefs_mountpoint, image_root: Path, dest_root: Path):
     (image_root / "thing.txt").write_text("I am a mongoose", encoding="utf-8")
     with pytest.raises(BadCefsImage, match="Found an unexpected file"):
-        CefsImage(directory=image_root, cefs_root=cefs_root)
+        CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
 
 
-def test_cefs_can_add_metadata(cefs_root: Path, image_root: Path):
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+def test_cefs_can_add_metadata(cefs_mountpoint, image_root: Path):
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
     root.add_metadata("I am metadata")
     assert root.metadata == ["I am metadata"]
 
 
-def test_cefs_can_render_metadata(cefs_root: Path, image_root: Path, dest_root: Path):
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+def test_cefs_can_render_metadata(cefs_mountpoint, image_root: Path, dest_root: Path):
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
     root.add_metadata("Line 1")
     root.add_metadata("Line 2")
     root.add_metadata("Line 3")
@@ -83,23 +83,23 @@ def test_cefs_can_render_metadata(cefs_root: Path, image_root: Path, dest_root:
     assert (dest_root / METADATA_FILENAME).read_text(encoding="utf-8") == "Line 1\nLine 2\nLine 3\n"
 
 
-def test_cefs_can_render_existing_links(cefs_root: Path, image_root: Path, dest_root: Path):
-    (image_root / "thing").symlink_to(cefs_root / "thing")
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
+def test_cefs_can_render_existing_links(cefs_mountpoint, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_mountpoint / "thing")
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
     root.render_to(dest_root)
-    assert (dest_root / "thing").readlink() == Path(cefs_root / "thing")
+    assert (dest_root / "thing").readlink() == Path(cefs_mountpoint / "thing")
 
 
-def test_cefs_can_render_new_links(cefs_root: Path, image_root: Path, dest_root: Path):
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
-    root.link_path(Path("something"), cefs_root / "something")
+def test_cefs_can_render_new_links(cefs_mountpoint, image_root: Path, dest_root: Path):
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+    root.link_path(Path("something"), cefs_mountpoint / "something")
     root.render_to(dest_root)
-    assert (dest_root / "something").readlink() == Path(cefs_root / "something")
+    assert (dest_root / "something").readlink() == Path(cefs_mountpoint / "something")
 
 
-def test_cefs_can_replace_existing_links(cefs_root: Path, image_root: Path, dest_root: Path):
-    (image_root / "thing").symlink_to(cefs_root / "thing")
-    root = CefsImage(directory=image_root, cefs_root=cefs_root)
-    root.link_path(Path("thing"), cefs_root / "new")
+def test_cefs_can_replace_existing_links(cefs_mountpoint, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_mountpoint / "thing")
+    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+    root.link_path(Path("thing"), cefs_mountpoint / "new")
     root.render_to(dest_root)
-    assert (dest_root / "thing").readlink() == Path(cefs_root / "new")
+    assert (dest_root / "thing").readlink() == Path(cefs_mountpoint / "new")

From 396f77520aee510b755f252e239e3126a216a8fd Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 08:33:15 -0500
Subject: [PATCH 006/161] Look for any python but...not a good solution to
 'what about python3.8 and below'

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 58fc26eeb..70772da1b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 .NOTPARALLEL:
 
-PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin bash -c "command -v python3.10 || command -v python3.9 || echo .python3.9-not-found")
+PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin bash -c "command -v python3.10 || command -v python3.9 || command -v python3 || echo .python-not-found")
 VIRTUALENV?=.env
 VIRTUALENV_DONE:=$(VIRTUALENV)/.done
 export PYTHONPATH=$(CURDIR)/bin

From 5cf9df004255cd7506ecca3554f27062d3e78fee Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 08:35:13 -0500
Subject: [PATCH 007/161] Revert "Look for any python but...not a good solution
 to 'what about python3.8 and below'"

This reverts commit 396f77520aee510b755f252e239e3126a216a8fd.
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 70772da1b..58fc26eeb 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 .NOTPARALLEL:
 
-PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin bash -c "command -v python3.10 || command -v python3.9 || command -v python3 || echo .python-not-found")
+PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin bash -c "command -v python3.10 || command -v python3.9 || echo .python3.9-not-found")
 VIRTUALENV?=.env
 VIRTUALENV_DONE:=$(VIRTUALENV)/.done
 export PYTHONPATH=$(CURDIR)/bin

From f3afbccec477238cc965824cff33ee554fd3724a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 08:38:37 -0500
Subject: [PATCH 008/161] arg

---
 .github/workflows/python-tests.yaml | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
index ea115be00..8daa61e7b 100644
--- a/.github/workflows/python-tests.yaml
+++ b/.github/workflows/python-tests.yaml
@@ -5,12 +5,23 @@ on: [ push ]
 jobs:
   build:
     runs-on: ubuntu-20.04
+    strategy:
+        matrix:
+            python-version: [ 3.9, 3.10 ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: test
+        run: |
+            which python
+            which python3
+            python --version
+            python3 --version
+            which python3.10
       - name: Install dependencies
         run: make ce
       - name: Check static

From 20da37fd5a6d7c8c35db4c0151d618d82ebabee4 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 08:39:48 -0500
Subject: [PATCH 009/161] arg

---
 .github/workflows/python-tests.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
index 8daa61e7b..263b7df1e 100644
--- a/.github/workflows/python-tests.yaml
+++ b/.github/workflows/python-tests.yaml
@@ -7,13 +7,13 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
         matrix:
-            python-version: [ 3.9, 3.10 ]
+            python-version: [ "3.9", "3.10" ]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "${{ matrix.python-version }}"
           cache: pip
       - name: test
         run: |

From f301769aa86757c82e9be262acf57f2450488428 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 08:46:50 -0500
Subject: [PATCH 010/161] Maybe will regret this

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 58fc26eeb..af76dacbd 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 .NOTPARALLEL:
 
-PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin bash -c "command -v python3.10 || command -v python3.9 || echo .python3.9-not-found")
+PYTHON:=$(shell env PATH=/bin:/usr/bin:/usr/local/bin:${PATH} bash -c "command -v python3.10 || command -v python3.9 || echo .python3.9-not-found")
 VIRTUALENV?=.env
 VIRTUALENV_DONE:=$(VIRTUALENV)/.done
 export PYTHONPATH=$(CURDIR)/bin

From 7d9b365a92f5d87d9936ba14ee5e207ef19e8a7a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 08:47:17 -0500
Subject: [PATCH 011/161] bleh

---
 .github/workflows/python-tests.yaml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
index 263b7df1e..c83f8b431 100644
--- a/.github/workflows/python-tests.yaml
+++ b/.github/workflows/python-tests.yaml
@@ -15,13 +15,6 @@ jobs:
         with:
           python-version: "${{ matrix.python-version }}"
           cache: pip
-      - name: test
-        run: |
-            which python
-            which python3
-            python --version
-            python3 --version
-            which python3.10
       - name: Install dependencies
         run: make ce
       - name: Check static

From 8256cf273acf0b045309d3b6330f107556d70aab Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 16:37:07 -0500
Subject: [PATCH 012/161] Oops

---
 bin/scripts/python_module_shim | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/scripts/python_module_shim b/bin/scripts/python_module_shim
index ddedf9fee..60fba5219 100755
--- a/bin/scripts/python_module_shim
+++ b/bin/scripts/python_module_shim
@@ -8,7 +8,6 @@ else
   ROOT_DIR=$(readlink -f "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/../..")
 fi
 MY_NAME=$(basename "$0")
-echo ${ROOT_DIR}
 
 if ! make -sq -C "${ROOT_DIR}" ce; then
     echo "python environment is out of date"

From 7e7dbcc2b5bf1baedbd9a388b51f3ac5346ad781 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 7 Sep 2022 18:16:25 -0500
Subject: [PATCH 013/161] todo

---
 bin/lib/cefs/__main__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 3f41daa48..5b9e2326c 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -145,6 +145,7 @@ def create_root(context: CliContext, root: Path):
 # TODO handle old symlinks of trunk (and do trunk etc!)
 # TODO should "cefs_mountpoint" be onfigurable.
 # TODO should there be a magic file? should metadata.txt by .metadata.txt or indeed just that special file?
+# TODO should probably check on mksquashfs cmdline flags like block size, my laptop used 131072 block size but I swear another machine used 4k
 
 
 def main():

From 84c6006ee6e7105f5a5a4a4a40b063b3930cab59 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 11 Sep 2022 09:17:01 -0500
Subject: [PATCH 014/161] some refacotring from laptop

---
 .idea/infra.iml          |  1 +
 bin/lib/ce_install.py    |  4 ++--
 bin/lib/cefs/__main__.py | 22 ++++++++++++++++++++--
 bin/lib/cefs/squash.py   | 37 ++++++++++++++++---------------------
 4 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/.idea/infra.iml b/.idea/infra.iml
index a69e0f2c1..423247063 100644
--- a/.idea/infra.iml
+++ b/.idea/infra.iml
@@ -6,6 +6,7 @@
       <excludeFolder url="file://$MODULE_DIR$/.deploy" />
       <excludeFolder url="file://$MODULE_DIR$/.dist" />
       <excludeFolder url="file://$MODULE_DIR$/venv" />
+      <excludeFolder url="file://$MODULE_DIR$/.env" />
     </content>
     <orderEntry type="jdk" jdkName="Python for infra" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py
index fdb55ec94..24080279e 100644
--- a/bin/lib/ce_install.py
+++ b/bin/lib/ce_install.py
@@ -383,7 +383,7 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
                 sys.exit(1)
 
     install_creator = SquashFsCreator(config=cefs_config)
-    with install_creator as tmp_path:
+    with install_creator.creation_path() as tmp_path:
         installation_context._staging_root = tmp_path / "staging"
         installation_context.destination = tmp_path
         _LOGGER.info("Installing everything to a temp dir")
@@ -408,7 +408,7 @@ def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image
 
     _LOGGER.info("Building new root fs")
     root_creator = SquashFsCreator(config=cefs_config)
-    with root_creator as tmp_path:
+    with root_creator.creation_path() as tmp_path:
         current_image.render_to(tmp_path)
 
     new_squashfs_cefs = root_creator.cefs_path
diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 5b9e2326c..3e36ea136 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -116,7 +116,7 @@ def create_image(context: CliContext):
 
 def _create_empty(config: CefsConfig) -> Path:
     creator = SquashFsCreator(config)
-    with creator as path:
+    with creator.creation_path() as path:
         image = CefsRootImage(cefs_mountpoint=config.mountpoint)
         image.add_metadata(f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}")
         image.render_to(path)
@@ -132,6 +132,19 @@ def create_root(context: CliContext, root: Path):
     CefsFsRoot.create(base_image=empty_path, fs_root=root, config=context.config)
 
 
+@cli.command
+@click.argument("root", type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
+@click.pass_obj
+def import_image(context: CliContext, root: Path):
+    """Import an existing filesystem ROOT."""
+    # DOESNT work on cefs images as they contain symlinks and...that would be bad for consolidation
+    # todo check it isn't?
+    # todo maybe support exclusions? `-ef` one line per file
+    creator = SquashFsCreator(context.config)
+    creator.import_existing_path(root)
+    click.echo(f"Imported new cefs image created at {creator.cefs_path}")
+
+
 # TODO how to start the whole thing off? make an empty root and manually symlink it in position?
 # TODO cases where we have a GIANT image that only exists because one dir is in it
 #  ie repacking and refreshing old images
@@ -145,7 +158,12 @@ def create_root(context: CliContext, root: Path):
 # TODO handle old symlinks of trunk (and do trunk etc!)
 # TODO should "cefs_mountpoint" be onfigurable.
 # TODO should there be a magic file? should metadata.txt by .metadata.txt or indeed just that special file?
-# TODO should probably check on mksquashfs cmdline flags like block size, my laptop used 131072 block size but I swear another machine used 4k
+# TODO should probably check on mksquashfs cmdline flags like block size, my laptop used 131072 block size but I swear
+#  another machine used 4k
+# TODO can add metadata directly with `--pseudofile` magic:
+#  https://github.com/plougher/squashfs-tools/blob/a5df5dc42c564d10c1aebf2063fc30c26850ddc3/USAGE#L261
+# TODO installation from ce_install of nightly things won't work as they try to remove things, and we need a solution
+#  for that
 
 
 def main():
diff --git a/bin/lib/cefs/squash.py b/bin/lib/cefs/squash.py
index ec15e0a08..5fec40e39 100644
--- a/bin/lib/cefs/squash.py
+++ b/bin/lib/cefs/squash.py
@@ -1,30 +1,30 @@
 from __future__ import annotations
 
-import shutil
+import contextlib
+import logging
 import subprocess
 from pathlib import Path
-from tempfile import TemporaryDirectory, mkdtemp
+from tempfile import TemporaryDirectory
+from typing import Optional, Iterable, Iterator
 
 from lib.cefs.config import CefsConfig
 
-
-import logging
-
 _LOGGER = logging.getLogger(__name__)
 
 
 class SquashFsCreator:
     def __init__(self, config: CefsConfig):
         self._config = config
-        self._sha = None
+        self._sha: Optional[str] = None
 
-    def _close(self):
+    def import_existing_path(self, path: Path):
+        assert self._sha is None
         with TemporaryDirectory(prefix="ce-squash-builder") as tmp_dir:
             tmp_sqfs = Path(tmp_dir) / "temp.sqfs"
             subprocess.check_call(
                 [
-                    "/usr/bin/mksquashfs",
-                    str(self._path),
+                    "mksquashfs",
+                    str(path),
                     str(tmp_sqfs),
                     "-all-root",
                     "-root-mode",
@@ -34,7 +34,7 @@ def _close(self):
                     "zstd",
                 ]
             )
-            self._sha, _filename = subprocess.check_output(["/usr/bin/shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            self._sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()
             if not self.image.exists():
                 _LOGGER.info("New squashfs image: %s", self.image)
                 tmp_sqfs.replace(self.image)
@@ -52,14 +52,9 @@ def cefs_path(self) -> Path:
         assert self._sha is not None
         return self._config.mountpoint / self._sha
 
-    def __enter__(self):
-        self._path = Path(mkdtemp(prefix="ce-install-temp"))
-        return self._path
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        try:
-            if not exc_type:
-                self._close()
-        finally:
-            subprocess.check_call(["chmod", "-R", "u+w", str(self._path)])
-            shutil.rmtree(self._path, ignore_errors=True)
+    @contextlib.contextmanager
+    def creation_path(self) -> Iterator[Path]:
+        with TemporaryDirectory(prefix="ce-install-temp") as temp_dir:
+            tmp_path = Path(temp_dir)
+            yield tmp_path
+            self.import_existing_path(tmp_path)

From ff1c254b37a70f069e220f81ddeddb8b0fc049aa Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 11 Sep 2022 18:36:39 -0500
Subject: [PATCH 015/161] More wip

---
 .idea/vcs.xml              |  6 ++++
 bin/lib/cefs/__main__.py   | 62 ++++++++++++++++++++++----------
 bin/lib/cefs/root.py       |  4 +--
 bin/lib/cefs/root_image.py | 19 +++++++---
 bin/test/cefs_test.py      | 72 +++++++++++++++++++++-----------------
 5 files changed, 105 insertions(+), 58 deletions(-)
 create mode 100644 .idea/vcs.xml

diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 000000000..94a25f7f4
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 3e36ea136..91deb9177 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -106,45 +106,69 @@ def install(context: CliContext):
     )
 
 
-@cli.command
+@cli.group
+def image():
+    """Create and manipulate raw squashfs images."""
+
+
+@image.command(name="create")
 @click.pass_obj
-def create_image(context: CliContext):
+def create_root(context: CliContext):
     """Create an empty image."""
     created_path = _create_empty(context.config)
     click.echo(f"Fresh new cefs image created at {created_path}")
 
 
+@image.command(name="import")
+@click.argument("root", type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
+@click.pass_obj
+def import_cmd(context: CliContext, root: Path):
+    """Import an existing filesystem ROOT."""
+    # DOESNT work on cefs images as they contain symlinks and...that would be bad for consolidation
+    # todo check it isn't?
+    # todo maybe support exclusions? `-ef` one line per file
+    creator = SquashFsCreator(context.config)
+    creator.import_existing_path(root)
+    click.echo(f"Imported new cefs image created at {creator.cefs_path}")
+
+
+@image.command
+@click.argument("root", type=click.Path(exists=True, dir_okay=True, path_type=Path), required=True)
+@click.pass_obj
+def info(context: CliContext, root: Path):
+    """Get information on the cefs root at ROOT."""
+    image = CefsRootImage(config=context.config, directory=root)
+    click.echo("Paths supported:")
+    for path in image.catalog:
+        click.echo(f"  {path}")
+    click.echo("Images used:")
+    for path in image.dependent_images:
+        click.echo(f"  {path}")
+
+
 def _create_empty(config: CefsConfig) -> Path:
     creator = SquashFsCreator(config)
     with creator.creation_path() as path:
-        image = CefsRootImage(cefs_mountpoint=config.mountpoint)
+        image = CefsRootImage(config=config)
         image.add_metadata(f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}")
         image.render_to(path)
     return creator.cefs_path
 
 
-@cli.command
+@cli.group
+def fs():
+    """Create and manipulate cefs filesystems."""
+
+
+@fs.command(name="create")
 @click.argument("root", type=click.Path(file_okay=False, dir_okay=False, writable=True, path_type=Path), required=True)
 @click.pass_obj
-def create_root(context: CliContext, root: Path):
-    """Create an empty cefs root at ROOT."""
+def create_fs_root(context: CliContext, root: Path):
+    """Create an empty cefs filesystem at ROOT."""
     empty_path = _create_empty(context.config)
     CefsFsRoot.create(base_image=empty_path, fs_root=root, config=context.config)
 
 
-@cli.command
-@click.argument("root", type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
-@click.pass_obj
-def import_image(context: CliContext, root: Path):
-    """Import an existing filesystem ROOT."""
-    # DOESNT work on cefs images as they contain symlinks and...that would be bad for consolidation
-    # todo check it isn't?
-    # todo maybe support exclusions? `-ef` one line per file
-    creator = SquashFsCreator(context.config)
-    creator.import_existing_path(root)
-    click.echo(f"Imported new cefs image created at {creator.cefs_path}")
-
-
 # TODO how to start the whole thing off? make an empty root and manually symlink it in position?
 # TODO cases where we have a GIANT image that only exists because one dir is in it
 #  ie repacking and refreshing old images
diff --git a/bin/lib/cefs/root.py b/bin/lib/cefs/root.py
index 5a9bc338f..7f5f43626 100644
--- a/bin/lib/cefs/root.py
+++ b/bin/lib/cefs/root.py
@@ -47,7 +47,7 @@ def create(cls, base_image: Path, fs_root: Path, config: CefsConfig) -> CefsFsRo
         if not base_image.is_dir():
             raise RuntimeError("Missing base image")
         # Construct just to ensure it's a valid image.
-        CefsRootImage(cefs_mountpoint=config.mountpoint, directory=base_image)
+        CefsRootImage(config=config, directory=base_image)
         fs_root.parent.mkdir(parents=True, exist_ok=True)
         # TODO append a log?
         fs_root.symlink_to(base_image, target_is_directory=True)
@@ -62,7 +62,7 @@ def image_root(self) -> Path:
         return self._image_root
 
     def read_image(self) -> CefsRootImage:
-        image = CefsRootImage(cefs_mountpoint=self._config.mountpoint, directory=self._image_root)
+        image = CefsRootImage(config=self._config, directory=self._image_root)
         image.add_metadata(f"Information read from root image {self._image_root}")
         return image
 
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index c4766e645..e68211ac2 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -4,6 +4,8 @@
 from pathlib import Path
 from typing import Optional, Dict, List, Mapping
 
+from lib.cefs.config import CefsConfig
+
 _LOGGER = logging.getLogger(__name__)
 METADATA_FILENAME = "metadata.txt"
 
@@ -22,8 +24,8 @@ class CefsRootImage:
     metadata. TODO naming
     """
 
-    def __init__(self, *, cefs_mountpoint: Path, directory: Optional[Path] = None):
-        self._cefs_mountpoint = cefs_mountpoint.resolve(strict=True)
+    def __init__(self, *, config: CefsConfig, directory: Optional[Path] = None):
+        self._config = config
         self._catalog: Dict[Path, Path] = {}
         self._metadata: List[str] = []
 
@@ -36,7 +38,7 @@ def _recurse_find_paths(self, path: Path, root_path: Path):
             relative = entry.relative_to(root_path)
             if entry.is_symlink():
                 link = entry.readlink()
-                if not link.is_relative_to(self._cefs_mountpoint):
+                if not link.is_relative_to(self._config.mountpoint):
                     raise BadCefsImage(f"Found a symlink that wasn't a symlink to cefs: {entry} links to {link}")
                 _LOGGER.debug("Found existing %s -> %s", relative, link)
                 self._catalog[relative] = link
@@ -54,7 +56,7 @@ def add_metadata(self, metadata: str) -> None:
         self._metadata.append(metadata)
 
     def link_path(self, subdir: Path, cefs_link: Path):
-        if not cefs_link.is_relative_to(self._cefs_mountpoint):
+        if not cefs_link.is_relative_to(self._config.mountpoint):
             raise BadCefsLink(f"Link is not relative to cefs: {cefs_link}")
         self._catalog[subdir] = cefs_link
 
@@ -62,6 +64,15 @@ def link_path(self, subdir: Path, cefs_link: Path):
     def catalog(self) -> Mapping[Path, Path]:
         return self._catalog
 
+    @property
+    def dependent_images(self) -> List[Path]:
+        return sorted(
+            set(
+                (self._config.image_root / x.relative_to(self._config.mountpoint).parts[0]).with_suffix(".sqfs")
+                for x in self._catalog.values()
+            )
+        )
+
     @property
     def metadata(self) -> List[str]:
         return self._metadata
diff --git a/bin/test/cefs_test.py b/bin/test/cefs_test.py
index 7c750e122..0e340a4fc 100644
--- a/bin/test/cefs_test.py
+++ b/bin/test/cefs_test.py
@@ -2,6 +2,7 @@
 
 import pytest
 
+from lib.cefs.config import CefsConfig
 from lib.cefs.root_image import CefsRootImage, METADATA_FILENAME, BadCefsImage
 
 
@@ -26,56 +27,61 @@ def dest_root(tmp_path) -> Path:
     return result
 
 
-def test_cefs_root_should_handle_no_initial_path(cefs_mountpoint):
-    root = CefsRootImage(cefs_mountpoint=cefs_mountpoint)
+@pytest.fixture
+def cefs_config(cefs_mountpoint, image_root) -> CefsConfig:
+    return CefsConfig(mountpoint=cefs_mountpoint, image_root=dest_root)
+
+
+def test_cefs_root_should_handle_no_initial_path(cefs_config):
+    root = CefsRootImage(config=cefs_config)
     assert not root.catalog
     assert not root.metadata
 
 
-def test_cefs_root_should_handle_empty_dirs(cefs_mountpoint, image_root: Path):
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+def test_cefs_root_should_handle_empty_dirs(cefs_config: CefsConfig, image_root: Path):
+    root = CefsRootImage(directory=image_root, config=cefs_config)
     assert not root.catalog
     assert not root.metadata
 
 
-def test_cefs_can_read_existing_images(cefs_mountpoint, image_root: Path, dest_root: Path):
-    (image_root / "thing").symlink_to(cefs_mountpoint / "thing")
+def test_cefs_can_read_existing_images(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_config.mountpoint / "thing")
     (image_root / "some" / "sub").mkdir(parents=True)
-    (image_root / "some" / "sub" / "directory").symlink_to(cefs_mountpoint / "thing" / "in" / "subdir")
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+    (image_root / "some" / "sub" / "directory").symlink_to(cefs_config.mountpoint / "thing" / "in" / "subdir")
+    root = CefsRootImage(directory=image_root, config=cefs_config)
     assert root.catalog == {
-        Path("thing"): cefs_mountpoint / "thing",
-        Path("some") / "sub" / "directory": cefs_mountpoint / "thing" / "in" / "subdir",
+        Path("thing"): cefs_config.mountpoint / "thing",
+        Path("some") / "sub" / "directory": cefs_config.mountpoint / "thing" / "in" / "subdir",
     }
     assert not root.metadata
 
 
-def test_cefs_can_read_existing_images_metadata(cefs_mountpoint, image_root: Path, dest_root: Path):
+def test_cefs_can_read_existing_images_metadata(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
     (image_root / METADATA_FILENAME).write_text("Meta\nData\nFTW\n", encoding="utf-8")
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+    root = CefsRootImage(directory=image_root, config=cefs_config)
     assert root.metadata == ["Meta", "Data", "FTW"]
 
 
-def test_cefs_is_upset_by_links_to_non_cefs(cefs_mountpoint, image_root: Path, dest_root: Path):
+def test_cefs_is_upset_by_links_to_non_cefs(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
     (image_root / "thing").symlink_to(dest_root)  # just a valid path not in cefs
     with pytest.raises(BadCefsImage, match="Found a symlink that wasn't a symlink to cefs"):
-        CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+        CefsRootImage(directory=image_root, config=cefs_config)
 
 
-def test_cefs_is_upset_by_random_files(cefs_mountpoint, image_root: Path, dest_root: Path):
+def test_cefs_is_upset_by_random_files(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
     (image_root / "thing.txt").write_text("I am a mongoose", encoding="utf-8")
     with pytest.raises(BadCefsImage, match="Found an unexpected file"):
-        CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+        CefsRootImage(directory=image_root, config=cefs_config)
 
 
-def test_cefs_can_add_metadata(cefs_mountpoint, image_root: Path):
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+def test_cefs_can_add_metadata(cefs_config: CefsConfig, image_root: Path):
+    root = CefsRootImage(directory=image_root, config=cefs_config)
     root.add_metadata("I am metadata")
     assert root.metadata == ["I am metadata"]
 
 
-def test_cefs_can_render_metadata(cefs_mountpoint, image_root: Path, dest_root: Path):
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+def test_cefs_can_render_metadata(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+    root = CefsRootImage(directory=image_root, config=cefs_config)
     root.add_metadata("Line 1")
     root.add_metadata("Line 2")
     root.add_metadata("Line 3")
@@ -83,23 +89,23 @@ def test_cefs_can_render_metadata(cefs_mountpoint, image_root: Path, dest_root:
     assert (dest_root / METADATA_FILENAME).read_text(encoding="utf-8") == "Line 1\nLine 2\nLine 3\n"
 
 
-def test_cefs_can_render_existing_links(cefs_mountpoint, image_root: Path, dest_root: Path):
-    (image_root / "thing").symlink_to(cefs_mountpoint / "thing")
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
+def test_cefs_can_render_existing_links(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_config.mountpoint / "thing")
+    root = CefsRootImage(directory=image_root, config=cefs_config)
     root.render_to(dest_root)
-    assert (dest_root / "thing").readlink() == Path(cefs_mountpoint / "thing")
+    assert (dest_root / "thing").readlink() == Path(cefs_config.mountpoint / "thing")
 
 
-def test_cefs_can_render_new_links(cefs_mountpoint, image_root: Path, dest_root: Path):
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
-    root.link_path(Path("something"), cefs_mountpoint / "something")
+def test_cefs_can_render_new_links(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+    root = CefsRootImage(directory=image_root, config=cefs_config)
+    root.link_path(Path("something"), cefs_config.mountpoint / "something")
     root.render_to(dest_root)
-    assert (dest_root / "something").readlink() == Path(cefs_mountpoint / "something")
+    assert (dest_root / "something").readlink() == Path(cefs_config.mountpoint / "something")
 
 
-def test_cefs_can_replace_existing_links(cefs_mountpoint, image_root: Path, dest_root: Path):
-    (image_root / "thing").symlink_to(cefs_mountpoint / "thing")
-    root = CefsRootImage(directory=image_root, cefs_mountpoint=cefs_mountpoint)
-    root.link_path(Path("thing"), cefs_mountpoint / "new")
+def test_cefs_can_replace_existing_links(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+    (image_root / "thing").symlink_to(cefs_config.mountpoint / "thing")
+    root = CefsRootImage(directory=image_root, config=cefs_config)
+    root.link_path(Path("thing"), cefs_config.mountpoint / "new")
     root.render_to(dest_root)
-    assert (dest_root / "thing").readlink() == Path(cefs_mountpoint / "new")
+    assert (dest_root / "thing").readlink() == Path(cefs_config.mountpoint / "new")

From 69cb7df69e33c3441aaac5b791dd0e79662b5ee1 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 11 Sep 2022 19:42:32 -0500
Subject: [PATCH 016/161] More wip

---
 .idea/runConfigurations/bin_lib_cefs.xml | 24 ++++++++++
 bin/lib/cefs/__main__.py                 | 18 ++++++++
 bin/lib/cefs/root_image.py               | 58 +++++++++++++++++++++++-
 3 files changed, 99 insertions(+), 1 deletion(-)
 create mode 100644 .idea/runConfigurations/bin_lib_cefs.xml

diff --git a/.idea/runConfigurations/bin_lib_cefs.xml b/.idea/runConfigurations/bin_lib_cefs.xml
new file mode 100644
index 000000000..ee6a4c119
--- /dev/null
+++ b/.idea/runConfigurations/bin_lib_cefs.xml
@@ -0,0 +1,24 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="bin.lib.cefs" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
+    <module name="infra" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
+    </envs>
+    <option name="SDK_HOME" value="$PROJECT_DIR$/.env/bin/python3" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/bin/lib/cefs" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <option name="SCRIPT_NAME" value="bin.lib.cefs" />
+    <option name="PARAMETERS" value="fs consolidate /opt/compiler-explorer" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="true" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>
\ No newline at end of file
diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 91deb9177..b31d6c8c0 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -169,6 +169,22 @@ def create_fs_root(context: CliContext, root: Path):
     CefsFsRoot.create(base_image=empty_path, fs_root=root, config=context.config)
 
 
+@fs.command
+@click.argument("root", type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
+@click.pass_obj
+def consolidate(context: CliContext, root: Path):
+    """Consolidate a FS root into a single layer."""
+    cefs_root = CefsFsRoot(fs_root=root, config=context.config)
+    cefs_image = cefs_root.read_image()
+    cefs_image.consolidate()
+    _LOGGER.info("Building new root fs")
+    root_creator = SquashFsCreator(config=context.config)
+    with root_creator.creation_path() as tmp_path:
+        cefs_image.render_to(tmp_path)
+    # TODO if not changed by something else in the mean time...
+    cefs_root.update(root_creator.cefs_path)
+
+
 # TODO how to start the whole thing off? make an empty root and manually symlink it in position?
 # TODO cases where we have a GIANT image that only exists because one dir is in it
 #  ie repacking and refreshing old images
@@ -188,6 +204,8 @@ def create_fs_root(context: CliContext, root: Path):
 #  https://github.com/plougher/squashfs-tools/blob/a5df5dc42c564d10c1aebf2063fc30c26850ddc3/USAGE#L261
 # TODO installation from ce_install of nightly things won't work as they try to remove things, and we need a solution
 #  for that
+# TODO installation of libraries
+# TODO consolidation and other fs commands that take a while need to be defensive about changes while they're going.
 
 
 def main():
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index e68211ac2..f20df6c4b 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import logging
+import subprocess
 from pathlib import Path
-from typing import Optional, Dict, List, Mapping
+from tempfile import TemporaryDirectory
+from typing import Optional, Dict, List, Mapping, Iterator
 
 from lib.cefs.config import CefsConfig
 
@@ -84,3 +86,57 @@ def render_to(self, destination: Path) -> None:
             source_dir = destination / entry
             source_dir.parent.mkdir(parents=True, exist_ok=True)
             source_dir.symlink_to(dest)
+
+    def _items_for(self, directory: Path, source_root: Path, dest_root: Path) -> Iterator[str]:
+        for item in directory.iterdir():
+            rel_path = dest_root / item.relative_to(source_root)
+            stat = item.lstat()
+            mode = f"0{oct(stat.st_mode & 0o7777)[2:]}"
+            uid = stat.st_uid
+            gid = stat.st_gid
+            if item.is_dir():
+                yield f'dir "{rel_path}" {mode} {uid} {gid}'
+                yield from self._items_for(directory=item, source_root=source_root, dest_root=dest_root)
+            elif item.is_file():
+                # TODO better hope there's no special characters as gensquashfs won't let the <location> have any...
+                yield f'file "{rel_path}" {mode} {uid} {gid} {item}'
+            elif item.is_symlink():
+                yield f'slink "{rel_path}" {mode} {uid} {gid} {item.readlink()}'
+            else:
+                raise RuntimeError(f"oh no {item}")  # TODO
+
+    def consolidate(self) -> None:
+        # TODO what if it's already one image? could still be consolidated e.g. if things were deleted in base image
+        # this might compact
+        # TODO put in squashfs build.py? or at least move functionality there
+        with TemporaryDirectory(prefix="ce-consolidate") as tmp_dir:
+            tmp_path = Path(tmp_dir)
+            tmp_sqfs = tmp_path / "temp.sqfs"
+            tmp_packfile = tmp_path / "packfile"
+            with tmp_packfile.open("w", encoding="utf-8") as tmp_packfile_file:
+                for entry, dest in self.catalog.items():
+                    _LOGGER.info("Finding things to consolidate: %s->%s", entry, dest)
+                    dest = dest.resolve(strict=True)
+                    for item in self._items_for(directory=dest, source_root=dest, dest_root=entry):
+                        tmp_packfile_file.write(f"{item}\n")
+            _LOGGER.info("Consolidating...")
+            subprocess.check_call(
+                [
+                    "gensquashfs",
+                    str(tmp_sqfs),
+                    "--pack-file",
+                    str(tmp_packfile),
+                    "--compressor",
+                    "zstd",
+                ]
+            )
+            sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            image = self._config.image_root / f"{sha}.sqfs"
+            if not image.exists():
+                _LOGGER.info("New squashfs image: %s", image)
+                tmp_sqfs.replace(image)
+            else:
+                _LOGGER.info("Existing: %s", image)
+                tmp_sqfs.unlink()
+        dest_image = self._config.mountpoint / sha
+        self._catalog = {entry: dest_image / entry for entry, dest in self._catalog.items()}

From 9bde0e1f752a89bcb40d9108919d10afa59fb4b0 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 11 Sep 2022 22:08:27 -0500
Subject: [PATCH 017/161] Add some squash and autofs tooling

---
 setup-admin.sh  | 6 +++++-
 setup-common.sh | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/setup-admin.sh b/setup-admin.sh
index 2c776b258..a7cba4dfc 100755
--- a/setup-admin.sh
+++ b/setup-admin.sh
@@ -14,7 +14,11 @@ fi
 
 env EXTRA_NFS_ARGS="" "${DIR}/setup-common.sh"
 
-apt -y install mosh fish jq cronic subversion upx gdb autojump zlib1g-dev m4 python3 python3-venv python3.8 python3.8-venv libc6-dev-i386
+apt -y install \
+  mosh fish jq cronic subversion upx gdb autojump \
+  zlib1g-dev m4 \
+  python3 python3-venv python3.8 python3.8-venv libc6-dev-i386 \
+  squashfs-tools-ng
 chsh ubuntu -s /usr/bin/fish
 
 cd /home/ubuntu/infra
diff --git a/setup-common.sh b/setup-common.sh
index e913461e6..61db00b17 100755
--- a/setup-common.sh
+++ b/setup-common.sh
@@ -19,6 +19,7 @@ wait_for_apt
 apt-get -y update
 apt-get -y upgrade --force-yes
 apt-get -y install \
+  autofs \
   jq \
   libc6-arm64-cross \
   libdatetime-perl \

From da42017b29484f1cc6afdec6a913ea6b837b4d30 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 12 Sep 2022 07:39:22 -0500
Subject: [PATCH 018/161] python3.9 is minimum now

---
 setup-common.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup-common.sh b/setup-common.sh
index 61db00b17..a7187c391 100755
--- a/setup-common.sh
+++ b/setup-common.sh
@@ -26,8 +26,9 @@ apt-get -y install \
   libtinfo5 \
   libwww-perl \
   nfs-common \
+  python3.9 \
   python3-pip \
-  python3-venv \
+  python3.9-venv \
   qemu-user-static \
   ssmtp \
   unzip \

From 583866316dba0285c497ca7b99cfedbb9d0fef98 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 13 Sep 2022 07:56:04 -0500
Subject: [PATCH 019/161] Horrible copy paste fest leading to import

---
 bin/lib/cefs/__main__.py   | 26 +++++++++++++++++++++++-
 bin/lib/cefs/root_image.py | 41 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 64 insertions(+), 3 deletions(-)

diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index b31d6c8c0..17f2cbc41 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -8,7 +8,7 @@
 import sys
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
+from typing import Optional, List
 
 import click
 
@@ -185,6 +185,29 @@ def consolidate(context: CliContext, root: Path):
     cefs_root.update(root_creator.cefs_path)
 
 
+@fs.command(name="import")
+@click.option(
+    "--relative-to",
+    type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
+    required=True,
+    help="import relative to PATH",
+)
+@click.argument("root", type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
+@click.argument("directory", nargs=-1, type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
+@click.pass_obj
+def import_(context: CliContext, root: Path, relative_to: Path, directory: List[Path]):
+    """Import existing directories into a fs."""
+    cefs_root = CefsFsRoot(fs_root=root, config=context.config)
+    cefs_image = cefs_root.read_image()
+    cefs_image.import_existing(relative_to, directory)
+    _LOGGER.info("Building new root fs")
+    root_creator = SquashFsCreator(config=context.config)
+    with root_creator.creation_path() as tmp_path:
+        cefs_image.render_to(tmp_path)
+    # TODO if not changed by something else in the mean time...
+    cefs_root.update(root_creator.cefs_path)
+
+
 # TODO how to start the whole thing off? make an empty root and manually symlink it in position?
 # TODO cases where we have a GIANT image that only exists because one dir is in it
 #  ie repacking and refreshing old images
@@ -206,6 +229,7 @@ def consolidate(context: CliContext, root: Path):
 #  for that
 # TODO installation of libraries
 # TODO consolidation and other fs commands that take a while need to be defensive about changes while they're going.
+# TODO /opt/cefs/images instead? then /opt/cefs/fs symlinks to all fs roots?
 
 
 def main():
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index f20df6c4b..a573bb1cd 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -4,7 +4,7 @@
 import subprocess
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Optional, Dict, List, Mapping, Iterator
+from typing import Optional, Dict, List, Mapping, Iterator, Iterable
 
 from lib.cefs.config import CefsConfig
 
@@ -109,7 +109,7 @@ def consolidate(self) -> None:
         # TODO what if it's already one image? could still be consolidated e.g. if things were deleted in base image
         # this might compact
         # TODO put in squashfs build.py? or at least move functionality there
-        with TemporaryDirectory(prefix="ce-consolidate") as tmp_dir:
+        with TemporaryDirectory(prefix="cefs-consolidate") as tmp_dir:
             tmp_path = Path(tmp_dir)
             tmp_sqfs = tmp_path / "temp.sqfs"
             tmp_packfile = tmp_path / "packfile"
@@ -140,3 +140,40 @@ def consolidate(self) -> None:
                 tmp_sqfs.unlink()
         dest_image = self._config.mountpoint / sha
         self._catalog = {entry: dest_image / entry for entry, dest in self._catalog.items()}
+
+    def import_existing(self, root_dir: Path, subdirs: Iterable[Path]) -> None:
+        with TemporaryDirectory(prefix="cefs-import") as tmp_dir:
+            tmp_path = Path(tmp_dir)
+            tmp_sqfs = tmp_path / "temp.sqfs"
+            tmp_packfile = tmp_path / "packfile"
+            new_entries = []
+            with tmp_packfile.open("w", encoding="utf-8") as tmp_packfile_file:
+                for to_import in subdirs:
+                    entry = to_import.relative_to(root_dir)
+                    new_entries.append(entry)
+                    _LOGGER.info("Scanning for import: %s->%s", entry, to_import)
+                    to_import = to_import.resolve(strict=True)
+                    for item in self._items_for(directory=to_import, source_root=to_import, dest_root=entry):
+                        tmp_packfile_file.write(f"{item}\n")
+            _LOGGER.info("Importing...")
+            subprocess.check_call(
+                [
+                    "gensquashfs",
+                    str(tmp_sqfs),
+                    "--pack-file",
+                    str(tmp_packfile),
+                    "--compressor",
+                    "zstd",
+                ]
+            )
+            sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            image = self._config.image_root / f"{sha}.sqfs"
+            if not image.exists():
+                _LOGGER.info("New squashfs image: %s", image)
+                tmp_sqfs.replace(image)
+            else:
+                _LOGGER.info("Existing: %s", image)
+                tmp_sqfs.unlink()
+        dest_image = self._config.mountpoint / sha
+        for new_entry in new_entries:
+            self._catalog[new_entry] = dest_image / new_entry

From 51dbec47ebd307fe283108e147f46c0c65c88f3a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 13 Sep 2022 18:31:59 -0500
Subject: [PATCH 020/161] More WIP

---
 bin/lib/ce_install.py      | 11 +++++++----
 bin/lib/cefs/__main__.py   | 13 ++++++++++---
 bin/lib/cefs/root_image.py | 12 +++++++++++-
 bin/lib/installation.py    | 10 ++++++++++
 4 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py
index 81e4d1ee0..100724663 100644
--- a/bin/lib/ce_install.py
+++ b/bin/lib/ce_install.py
@@ -207,7 +207,7 @@ def cli(
         file_handler.setFormatter(formatter)
         root_logger.addHandler(file_handler)
     if not log or log_to_console:
-        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler = logging.StreamHandler(sys.stderr)
         console_handler.setFormatter(formatter)
         root_logger.addHandler(console_handler)
     context = InstallationContext(
@@ -229,12 +229,15 @@ def cli(
 
 @cli.command(name="list")
 @click.pass_obj
+@click.option("--json", is_flag=True, help="Output in JSON format")
+@click.option("--installed-only", is_flag=True, help="Only output installed targets")
 @click.argument("filter_", metavar="FILTER", nargs=-1)
-def list_cmd(context: CliContext, filter_: List[str]):
+def list_cmd(context: CliContext, filter_: List[str], json: bool, installed_only: bool):
     """List installation targets matching FILTER."""
-    print("Installation candidates:")
     for installable in context.get_installables(filter_):
-        print(installable.name)
+        if installed_only and not installable.is_installed():
+            continue
+        print(installable.to_json() if json else installable.name)
         _LOGGER.debug(installable)
 
 
diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 17f2cbc41..8e87f3dc0 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -190,16 +190,17 @@ def consolidate(context: CliContext, root: Path):
     "--relative-to",
     type=click.Path(file_okay=False, dir_okay=True, path_type=Path),
     required=True,
-    help="import relative to PATH",
+    help="Import relative to PATH",
 )
+@click.option("--replace/--no-replace", help="Replace (or not) anything in the path")
 @click.argument("root", type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
 @click.argument("directory", nargs=-1, type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
 @click.pass_obj
-def import_(context: CliContext, root: Path, relative_to: Path, directory: List[Path]):
+def import_(context: CliContext, root: Path, relative_to: Path, directory: List[Path], replace: bool):
     """Import existing directories into a fs."""
     cefs_root = CefsFsRoot(fs_root=root, config=context.config)
     cefs_image = cefs_root.read_image()
-    cefs_image.import_existing(relative_to, directory)
+    cefs_image.import_existing(relative_to, directory, replace=replace)
     _LOGGER.info("Building new root fs")
     root_creator = SquashFsCreator(config=context.config)
     with root_creator.creation_path() as tmp_path:
@@ -230,6 +231,12 @@ def import_(context: CliContext, root: Path, relative_to: Path, directory: List[
 # TODO installation of libraries
 # TODO consolidation and other fs commands that take a while need to be defensive about changes while they're going.
 # TODO /opt/cefs/images instead? then /opt/cefs/fs symlinks to all fs roots?
+# TODO rust libraries don't have install_path
+# TODO if we import (e.g.) `arm` wholesale, and then later add `arm/blah` as a sudirectory, need to "split" `arm` in the
+#  symlinks into its components
+# useful cmd? ce_install list gcc --json --installed-only \
+#   | jq -r .install_path \
+#   | xargs cefs fs import --relative-to /opt/compiler-explorer ~/ce
 
 
 def main():
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index a573bb1cd..805e60649 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -141,7 +141,7 @@ def consolidate(self) -> None:
         dest_image = self._config.mountpoint / sha
         self._catalog = {entry: dest_image / entry for entry, dest in self._catalog.items()}
 
-    def import_existing(self, root_dir: Path, subdirs: Iterable[Path]) -> None:
+    def import_existing(self, root_dir: Path, subdirs: Iterable[Path], replace: bool = False) -> None:
         with TemporaryDirectory(prefix="cefs-import") as tmp_dir:
             tmp_path = Path(tmp_dir)
             tmp_sqfs = tmp_path / "temp.sqfs"
@@ -149,7 +149,17 @@ def import_existing(self, root_dir: Path, subdirs: Iterable[Path]) -> None:
             new_entries = []
             with tmp_packfile.open("w", encoding="utf-8") as tmp_packfile_file:
                 for to_import in subdirs:
+                    if not to_import.absolute():
+                        to_import = root_dir / to_import
+                        if not to_import.is_dir():
+                            raise RuntimeError(f"{to_import} was not a directory when adding")
                     entry = to_import.relative_to(root_dir)
+                    if entry in self._catalog:
+                        if replace:
+                            _LOGGER.info("Replacing existing %s", entry)
+                        else:
+                            _LOGGER.info("Skipping existing %s", entry)
+                            continue
                     new_entries.append(entry)
                     _LOGGER.info("Scanning for import: %s->%s", entry, to_import)
                     to_import = to_import.resolve(strict=True)
diff --git a/bin/lib/installation.py b/bin/lib/installation.py
index 0521dc8e7..29cd87179 100644
--- a/bin/lib/installation.py
+++ b/bin/lib/installation.py
@@ -3,6 +3,7 @@
 import contextlib
 import functools
 import glob
+import json
 import logging
 import os
 import re
@@ -303,6 +304,9 @@ def set_rpath(self, elf_file: Path, rpath: str):
         subprocess.check_call([self.destination / "patchelf-0.8" / "src" / "patchelf", "--set-rpath", rpath, elf_file])
 
 
+SimpleJsonType = (int, float, str, bool)
+
+
 class Installable:
     _check_link: Optional[Callable[[], bool]]
     check_env: Dict
@@ -333,6 +337,12 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.after_stage_script = self.config_get("after_stage_script", [])
         self._logger = logging.getLogger(self.name)
 
+    def to_json_dict(self) -> Dict[str, Any]:
+        return {key: value for key, value in self.__dict__.items() if isinstance(value, SimpleJsonType)}
+
+    def to_json(self) -> str:
+        return json.dumps(self.to_json_dict())
+
     def _setup_check_exe(self, path_name: str) -> None:
         self.check_env = dict([x.replace("%PATH%", path_name).split("=", 1) for x in self.config_get("check_env", [])])
 

From 031b7d967f54db6d4b07b9ab14d19b14784c31d8 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 13 Sep 2022 18:35:20 -0500
Subject: [PATCH 021/161] zomg

---
 bin/lib/cefs/__main__.py   | 1 +
 bin/lib/cefs/root_image.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 8e87f3dc0..95fcd13d9 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -234,6 +234,7 @@ def import_(context: CliContext, root: Path, relative_to: Path, directory: List[
 # TODO rust libraries don't have install_path
 # TODO if we import (e.g.) `arm` wholesale, and then later add `arm/blah` as a sudirectory, need to "split" `arm` in the
 #  symlinks into its components
+# TODO  - can't move from one FS to another so should make sure temp is on dest drive or handle it.
 # useful cmd? ce_install list gcc --json --installed-only \
 #   | jq -r .install_path \
 #   | xargs cefs fs import --relative-to /opt/compiler-explorer ~/ce
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index 805e60649..ffb2717af 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -149,7 +149,7 @@ def import_existing(self, root_dir: Path, subdirs: Iterable[Path], replace: bool
             new_entries = []
             with tmp_packfile.open("w", encoding="utf-8") as tmp_packfile_file:
                 for to_import in subdirs:
-                    if not to_import.absolute():
+                    if not to_import.is_absolute():
                         to_import = root_dir / to_import
                         if not to_import.is_dir():
                             raise RuntimeError(f"{to_import} was not a directory when adding")

From 02d44a25f1eab8b27f139693d390621ff63ce462 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 14 Sep 2022 07:51:17 -0500
Subject: [PATCH 022/161] More TODOs

---
 bin/lib/cefs/__main__.py   | 5 +++--
 bin/lib/cefs/root_image.py | 3 +++
 requirements.txt           | 1 +
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 95fcd13d9..324ea3d63 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -11,6 +11,7 @@
 from typing import Optional, List
 
 import click
+import humanfriendly
 
 from lib.cefs.config import CefsConfig
 from lib.cefs.root import CefsFsRoot
@@ -142,8 +143,8 @@ def info(context: CliContext, root: Path):
     for path in image.catalog:
         click.echo(f"  {path}")
     click.echo("Images used:")
-    for path in image.dependent_images:
-        click.echo(f"  {path}")
+    for size, path in sorted(((path.stat().st_size, path) for path in image.dependent_images), reverse=True):
+        click.echo(f"  {path} ({humanfriendly.format_size(size, binary=True)})")
 
 
 def _create_empty(config: CefsConfig) -> Path:
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index ffb2717af..825fa0c55 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -88,6 +88,7 @@ def render_to(self, destination: Path) -> None:
             source_dir.symlink_to(dest)
 
     def _items_for(self, directory: Path, source_root: Path, dest_root: Path) -> Iterator[str]:
+        # TODO er, glob?
         for item in directory.iterdir():
             rel_path = dest_root / item.relative_to(source_root)
             stat = item.lstat()
@@ -108,6 +109,7 @@ def _items_for(self, directory: Path, source_root: Path, dest_root: Path) -> Ite
     def consolidate(self) -> None:
         # TODO what if it's already one image? could still be consolidated e.g. if things were deleted in base image
         # this might compact
+        # TODO keep some number of layers?
         # TODO put in squashfs build.py? or at least move functionality there
         with TemporaryDirectory(prefix="cefs-consolidate") as tmp_dir:
             tmp_path = Path(tmp_dir)
@@ -160,6 +162,7 @@ def import_existing(self, root_dir: Path, subdirs: Iterable[Path], replace: bool
                         else:
                             _LOGGER.info("Skipping existing %s", entry)
                             continue
+                    self.metadata.append(f"Importing {to_import} as {entry}")
                     new_entries.append(entry)
                     _LOGGER.info("Scanning for import: %s->%s", entry, to_import)
                     to_import = to_import.resolve(strict=True)
diff --git a/requirements.txt b/requirements.txt
index ba9fc5929..da8ef95ce 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@ awscli
 boto3>=1.17
 cachecontrol[filecache]
 click
+humanfriendly
 jinja2
 packaging
 paramiko

From 5511adc3e080396b33c96eaa196d954a70239400 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Thu, 22 Sep 2022 17:21:59 -0500
Subject: [PATCH 023/161] Implement remove

---
 bin/lib/cefs/__main__.py   | 25 +++++++++++++++++++++++++
 bin/lib/cefs/root_image.py |  7 +++++++
 2 files changed, 32 insertions(+)

diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 324ea3d63..5319f4aa6 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -186,6 +186,31 @@ def consolidate(context: CliContext, root: Path):
     cefs_root.update(root_creator.cefs_path)
 
 
+@fs.command(name="rm")
+@click.argument("root", type=click.Path(file_okay=False, dir_okay=True, path_type=Path), required=True)
+@click.argument("path", nargs=-1, type=click.Path(path_type=Path), required=True)
+@click.pass_obj
+def fs_remove(context: CliContext, root: Path, path: List[Path]):
+    """Remove PATH entries from a filesystem at ROOT."""
+    cefs_root = CefsFsRoot(fs_root=root, config=context.config)
+    cefs_image = cefs_root.read_image()
+    print(cefs_image.catalog)
+    for pathlet in path:
+        # TODO smart guess at "did you mean relative to the root of the fs if not absolute?
+        if not pathlet.is_absolute():
+            raise RuntimeError("yeah for now needs to be absolute")
+        if not pathlet.is_relative_to(root):
+            raise RuntimeError(f"{pathlet} not relative to {root}")
+        relative = pathlet.relative_to(root)
+        cefs_image.remove(relative)
+    _LOGGER.info("Building new root fs")
+    root_creator = SquashFsCreator(config=context.config)
+    with root_creator.creation_path() as tmp_path:
+        cefs_image.render_to(tmp_path)
+    # TODO if not changed by something else in the mean time...
+    cefs_root.update(root_creator.cefs_path)
+
+
 @fs.command(name="import")
 @click.option(
     "--relative-to",
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index 825fa0c55..8734b2b8d 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -75,6 +75,13 @@ def dependent_images(self) -> List[Path]:
             )
         )
 
+    def remove(self, relative_path: Path) -> None:
+        if relative_path not in self._catalog:
+            # TODO one day consider splitting if this is a _subdir_ of an existing image...etc
+            raise FileNotFoundError(f"{relative_path} is not in, or directly mapped as a subdir")
+        self.add_metadata(f"Removing mapping from {relative_path} to {self._catalog[relative_path]}")
+        del self._catalog[relative_path]
+
     @property
     def metadata(self) -> List[str]:
         return self._metadata

From b94eef9f39434f90ec2445c4c7282a9de3ea53ba Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 11 Oct 2022 17:23:59 -0500
Subject: [PATCH 024/161] First non-exploding lib installation

---
 bin/lib/ce_install.py              | 32 ++++++++++++++----------------
 bin/lib/cefs/__main__.py           | 16 +++++++--------
 bin/lib/cefs/root_image.py         |  1 +
 bin/lib/cefs/squash.py             |  3 +--
 bin/lib/installable/git.py         |  2 +-
 bin/lib/installable/installable.py |  3 +--
 bin/lib/installation_context.py    |  4 ++--
 7 files changed, 29 insertions(+), 32 deletions(-)

diff --git a/bin/lib/ce_install.py b/bin/lib/ce_install.py
index eac33d476..ec21a1e20 100644
--- a/bin/lib/ce_install.py
+++ b/bin/lib/ce_install.py
@@ -371,44 +371,42 @@ def squash(context: CliContext, filter_: List[str], force: bool, image_dir: Path
 )
 @click.argument("filter_", metavar="FILTER", nargs=-1)
 def buildroot(context: CliContext, filter_: List[str], force: bool, squash_image_root: Path, cefs_mountpoint: Path):
-    """Squash all things matching to a single root image."""
-
-    # check for things already installed in the _current_ root?
+    """Squash all things matching to a single layer."""
 
     installation_context = context.installation_context
     cefs_config = CefsConfig(mountpoint=cefs_mountpoint, image_root=squash_image_root)
     fs_root = CefsFsRoot(fs_root=installation_context.destination, config=cefs_config)
 
     current_image = fs_root.read_image()
+    to_install = []
     for installable in context.get_installables(filter_):
-        if force or installable.is_installed():
+        if force or installable.should_install():
+            to_install.append(installable)
             dest_path = installation_context.destination / installable.install_path
-            if not dest_path.is_symlink():
+            if dest_path.exists() and not dest_path.is_symlink():
                 _LOGGER.error("Found an installable that wasn't a symlink: %s", dest_path)
                 sys.exit(1)
 
     install_creator = SquashFsCreator(config=cefs_config)
     with install_creator.creation_path() as tmp_path:
+        _LOGGER.info("Installing everything to a temp dir: %s", tmp_path)
         installation_context.set_temp_destination(tmp_path)
-        _LOGGER.info("Installing everything to a temp dir")
         num_installed = 0
-        for installable in context.get_installables(filter_):
-            if Path(installable.install_path) not in current_image.catalog:
-                installable.install()
-                if not installable.is_installed():
-                    _LOGGER.error("%s installed OK, but doesn't appear as installed after", installable.name)
-                    sys.exit(1)
-                current_image.add_metadata(f"Installing {installable.install_path} from {installable}")
-                num_installed += 1
+        for installable in to_install:
+            installable.install()
+            if not installable.is_installed():
+                _LOGGER.error("%s installed OK, but doesn't appear as installed after", installable.name)
+                sys.exit(1)
+            current_image.add_metadata(f"Installing {installable.install_path} from {installable}")
+            num_installed += 1
 
     if not num_installed:
         click.echo("No changes: not updating base image")
         return
 
     new_squashfs_cefs = install_creator.cefs_path
-    for installable in context.get_installables(filter_):
-        if Path(installable.install_path) not in current_image.catalog:
-            current_image.link_path(Path(installable.install_path), new_squashfs_cefs / installable.install_path)
+    for installable in to_install:
+        current_image.link_path(Path(installable.install_path), new_squashfs_cefs / installable.install_path)
 
     _LOGGER.info("Building new root fs")
     root_creator = SquashFsCreator(config=cefs_config)
diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 5319f4aa6..99be35097 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -138,21 +138,23 @@ def import_cmd(context: CliContext, root: Path):
 @click.pass_obj
 def info(context: CliContext, root: Path):
     """Get information on the cefs root at ROOT."""
-    image = CefsRootImage(config=context.config, directory=root)
+    cefs_root_image = CefsRootImage(config=context.config, directory=root)
     click.echo("Paths supported:")
-    for path in image.catalog:
+    for path in cefs_root_image.catalog:
         click.echo(f"  {path}")
     click.echo("Images used:")
-    for size, path in sorted(((path.stat().st_size, path) for path in image.dependent_images), reverse=True):
+    for size, path in sorted(((path.stat().st_size, path) for path in cefs_root_image.dependent_images), reverse=True):
         click.echo(f"  {path} ({humanfriendly.format_size(size, binary=True)})")
 
 
 def _create_empty(config: CefsConfig) -> Path:
     creator = SquashFsCreator(config)
     with creator.creation_path() as path:
-        image = CefsRootImage(config=config)
-        image.add_metadata(f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}")
-        image.render_to(path)
+        cefs_root_image = CefsRootImage(config=config)
+        cefs_root_image.add_metadata(
+            f"Initial empty image created at {datetime.datetime.utcnow()} by {getpass.getuser()}"
+        )
+        cefs_root_image.render_to(path)
     return creator.cefs_path
 
 
@@ -248,8 +250,6 @@ def import_(context: CliContext, root: Path, relative_to: Path, directory: List[
 # TODO handle old symlinks of trunk (and do trunk etc!)
 # TODO should "cefs_mountpoint" be onfigurable.
 # TODO should there be a magic file? should metadata.txt by .metadata.txt or indeed just that special file?
-# TODO should probably check on mksquashfs cmdline flags like block size, my laptop used 131072 block size but I swear
-#  another machine used 4k
 # TODO can add metadata directly with `--pseudofile` magic:
 #  https://github.com/plougher/squashfs-tools/blob/a5df5dc42c564d10c1aebf2063fc30c26850ddc3/USAGE#L261
 # TODO installation from ce_install of nightly things won't work as they try to remove things, and we need a solution
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index 8734b2b8d..f9f571646 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -55,6 +55,7 @@ def _recurse_find_paths(self, path: Path, root_path: Path):
                 raise BadCefsImage(f"Found an unexpected entry: {entry}")
 
     def add_metadata(self, metadata: str) -> None:
+        # todo separate machine readable from human readable? add timestamp? json?
         self._metadata.append(metadata)
 
     def link_path(self, subdir: Path, cefs_link: Path):
diff --git a/bin/lib/cefs/squash.py b/bin/lib/cefs/squash.py
index 5fec40e39..ff9e1b328 100644
--- a/bin/lib/cefs/squash.py
+++ b/bin/lib/cefs/squash.py
@@ -5,7 +5,7 @@
 import subprocess
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Optional, Iterable, Iterator
+from typing import Optional, Iterator
 
 from lib.cefs.config import CefsConfig
 
@@ -26,7 +26,6 @@ def import_existing_path(self, path: Path):
                     "mksquashfs",
                     str(path),
                     str(tmp_sqfs),
-                    "-all-root",
                     "-root-mode",
                     "755",
                     "-progress",
diff --git a/bin/lib/installable/git.py b/bin/lib/installable/git.py
index 6c7207f05..9334760f5 100644
--- a/bin/lib/installable/git.py
+++ b/bin/lib/installable/git.py
@@ -19,7 +19,7 @@
 
 def _git_raw(logger: logging.Logger, cwd: Path, *git_args: Union[str, Path]) -> str:
     full_args = ["git"] + [str(x) for x in git_args]
-    _LOGGER.debug(shlex.join(full_args))
+    _LOGGER.debug("(in %s): %s", cwd, shlex.join(full_args))
     result = subprocess.check_output(full_args, cwd=cwd).decode("utf-8").strip()
     if result:
         logger.debug(" -> %s", result)
diff --git a/bin/lib/installable/installable.py b/bin/lib/installable/installable.py
index d7b89b7a4..1c6a2f690 100644
--- a/bin/lib/installable/installable.py
+++ b/bin/lib/installable/installable.py
@@ -195,10 +195,9 @@ def squash_to(self, destination_image: Path):
         self._logger.info("Squashing %s...", source_folder)
         self.install_context.check_call(
             [
-                "/usr/bin/mksquashfs",
+                "mksquashfs",
                 str(source_folder),
                 str(temp_image),
-                "-all-root",
                 "-progress",
                 "-comp",
                 "zstd",
diff --git a/bin/lib/installation_context.py b/bin/lib/installation_context.py
index a3fc3b6e8..3595816f2 100644
--- a/bin/lib/installation_context.py
+++ b/bin/lib/installation_context.py
@@ -79,8 +79,8 @@ def prior_installation(self) -> Path:
         return self._prior_installation
 
     def set_temp_destination(self, tmp_path: Path) -> None:
-        _staging_root = tmp_path / "staging"
-        _destination = tmp_path
+        self._staging_root = tmp_path / "staging"
+        self._destination = tmp_path
 
     @contextlib.contextmanager
     def new_staging_dir(self) -> Iterator[StagingDir]:

From 09bc5734cb85d38c7b0c6cb8a73c23421678871a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 25 Oct 2022 18:02:58 -0500
Subject: [PATCH 025/161] Bump alert

---
 terraform/cloudwatch.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/cloudwatch.tf b/terraform/cloudwatch.tf
index 06ffbfb10..af3e50f1a 100644
--- a/terraform/cloudwatch.tf
+++ b/terraform/cloudwatch.tf
@@ -165,7 +165,7 @@ resource "aws_cloudwatch_metric_alarm" "high_traffic" {
   alarm_description   = "A high amount of traffic: did we just get slashdotted?"
   evaluation_periods  = 3
   datapoints_to_alarm = 3
-  threshold           = 5000
+  threshold           = 7500
   metric_name         = "RequestCount"
   namespace           = "AWS/ApplicationELB"
   statistic           = "Sum"

From 51b729fe5c5444489bcedda33f66915094258488 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 29 Oct 2022 15:21:50 +0200
Subject: [PATCH 026/161] fix google benchmark 1.6.2

---
 bin/resources/googlebenchmark_export.h | 42 ++++++++++++++++++++++++++
 bin/yaml/libraries.yaml                |  4 ++-
 2 files changed, 45 insertions(+), 1 deletion(-)
 create mode 100644 bin/resources/googlebenchmark_export.h

diff --git a/bin/resources/googlebenchmark_export.h b/bin/resources/googlebenchmark_export.h
new file mode 100644
index 000000000..0ba3ce9c1
--- /dev/null
+++ b/bin/resources/googlebenchmark_export.h
@@ -0,0 +1,42 @@
+
+#ifndef BENCHMARK_EXPORT_H
+#define BENCHMARK_EXPORT_H
+
+#ifdef BENCHMARK_STATIC_DEFINE
+#  define BENCHMARK_EXPORT
+#  define BENCHMARK_NO_EXPORT
+#else
+#  ifndef BENCHMARK_EXPORT
+#    ifdef benchmark_EXPORTS
+        /* We are building this library */
+#      define BENCHMARK_EXPORT 
+#    else
+        /* We are using this library */
+#      define BENCHMARK_EXPORT 
+#    endif
+#  endif
+
+#  ifndef BENCHMARK_NO_EXPORT
+#    define BENCHMARK_NO_EXPORT 
+#  endif
+#endif
+
+#ifndef BENCHMARK_DEPRECATED
+#  define BENCHMARK_DEPRECATED __attribute__ ((__deprecated__))
+#endif
+
+#ifndef BENCHMARK_DEPRECATED_EXPORT
+#  define BENCHMARK_DEPRECATED_EXPORT BENCHMARK_EXPORT BENCHMARK_DEPRECATED
+#endif
+
+#ifndef BENCHMARK_DEPRECATED_NO_EXPORT
+#  define BENCHMARK_DEPRECATED_NO_EXPORT BENCHMARK_NO_EXPORT BENCHMARK_DEPRECATED
+#endif
+
+#if 0 /* DEFINE_NO_DEPRECATED */
+#  ifndef BENCHMARK_NO_DEPRECATED
+#    define BENCHMARK_NO_DEPRECATED
+#  endif
+#endif
+
+#endif /* BENCHMARK_EXPORT_H */
diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index b456ca4b5..23d3708f8 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -24,7 +24,9 @@ libraries:
       - 1.4.1
       - 1.5.0
       - 1.6.1
-      - 1.6.2
+      - name: 1.6.2
+        after_stage_script:
+        - cp {{resource_dir}}/googlebenchmark_export.h include/benchmark/export.h
       type: github
     benri:
       check_file: readme.md

From 77ec7dc78484d267ef7e13123cef99bb1d246fe7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Poulhi=C3=A8s?= <dkm@kataplop.net>
Date: Mon, 31 Oct 2022 10:36:54 +0100
Subject: [PATCH 027/161] Add Loongarch64 config (#853)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add 1 compiler for the Loongarch64 that has been introduced in GCC 12:
 https://gcc.gnu.org/gcc-12/changes.html

refs https://github.com/compiler-explorer/compiler-explorer/issues/4162

Signed-off-by: Marc Poulhiès <dkm@kataplop.net>
---
 bin/yaml/cpp.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index be91fa336..300df8a34 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -426,6 +426,12 @@ compilers:
             - 4.9.4
             - 9.5.0
             - 12.2.0
+        loongarch64:
+          arch_prefix: "{subdir}-unknown-linux-gnu"
+          check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"
+          subdir: loongarch64
+          targets:
+            - 12.2.0
         powerpc:
           arch_prefix: "{subdir}-unknown-linux-gnu"
           check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"

From a9b0b63cdc7683b34dfe194ab54b71ceac55610b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Poulhi=C3=A8s?= <dkm@kataplop.net>
Date: Mon, 31 Oct 2022 10:48:57 +0100
Subject: [PATCH 028/161] Fix trailing whitespaces

Looks like the whitespaces are triggering a checker: https://github.com/compiler-explorer/infra/actions/runs/3360157982/jobs/5569012150#step:5:47
---
 bin/resources/googlebenchmark_export.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/resources/googlebenchmark_export.h b/bin/resources/googlebenchmark_export.h
index 0ba3ce9c1..1cf10e035 100644
--- a/bin/resources/googlebenchmark_export.h
+++ b/bin/resources/googlebenchmark_export.h
@@ -9,15 +9,15 @@
 #  ifndef BENCHMARK_EXPORT
 #    ifdef benchmark_EXPORTS
         /* We are building this library */
-#      define BENCHMARK_EXPORT 
+#      define BENCHMARK_EXPORT
 #    else
         /* We are using this library */
-#      define BENCHMARK_EXPORT 
+#      define BENCHMARK_EXPORT
 #    endif
 #  endif
 
 #  ifndef BENCHMARK_NO_EXPORT
-#    define BENCHMARK_NO_EXPORT 
+#    define BENCHMARK_NO_EXPORT
 #  endif
 #endif
 

From c4a6c808b23c94e1365727fbcd963f0c702801dc Mon Sep 17 00:00:00 2001
From: Jan <20126033+JanLJL@users.noreply.github.com>
Date: Tue, 1 Nov 2022 03:46:42 +0100
Subject: [PATCH 029/161] updated OSACA to 0.4.12 (#848)

---
 bin/yaml/tools.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/yaml/tools.yaml b/bin/yaml/tools.yaml
index 0940ec50e..0303bf3a2 100644
--- a/bin/yaml/tools.yaml
+++ b/bin/yaml/tools.yaml
@@ -20,7 +20,7 @@ tools:
     package: osaca=={name}
     check_exe: bin/osaca --version
     targets:
-      - 0.4.11
+      - 0.4.12
   rustfmt:
     type: tarballs
     dir: rustfmt-{name}

From d3b25cef9660d718078a45f8e19c6f4b12986e95 Mon Sep 17 00:00:00 2001
From: Ryan Hirasaki <4690732+RyanHir@users.noreply.github.com>
Date: Mon, 31 Oct 2022 21:48:22 -0500
Subject: [PATCH 030/161] Add FRC 2023 Support (#851)

---
 bin/yaml/cpp.yaml | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 300df8a34..251137c32 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -184,19 +184,28 @@ compilers:
               - gcc-arm-none-eabi-5_4-2016q3 # name unused in this case
           - type: tarballs
             frc:
-              check_exe: roborio/bin/arm-frc{year}-linux-gnueabi-g++ --version
-              compression: gz
-              dir: arm/frc{year}-{name}
-              untar_dir: frc{year}
-              url: https://github.com/wpilibsuite/toolchain-builder/releases/download/v{year}-{build}/FRC-{year}-Linux-Toolchain-{name}.tar.gz
-              targets:
-                - year: 2019
-                  build: 3
-                  name: 6.3.0
-                - year: 2020
-                  build: 1
-                  name: 7.3.0
-                  url: https://github.com/wpilibsuite/roborio-toolchain/releases/download/v{year}-{build}/FRC-{year}-Linux-Toolchain-{name}.tar.gz
+              - check_exe: roborio/bin/arm-frc{year}-linux-gnueabi-g++ --version
+                compression: gz
+                dir: arm/frc{year}-{name}
+                untar_dir: frc{year}
+                url: https://github.com/wpilibsuite/toolchain-builder/releases/download/v{year}-{build}/FRC-{year}-Linux-Toolchain-{name}.tar.gz
+                targets:
+                  - year: 2019
+                    build: 3
+                    name: 6.3.0
+                  - year: 2020
+                    build: 1
+                    name: 7.3.0
+                    url: https://github.com/wpilibsuite/roborio-toolchain/releases/download/v{year}-{build}/FRC-{year}-Linux-Toolchain-{name}.tar.gz
+              - check_exe: bin/arm-frc{year}-linux-gnueabi-g++ --version
+                compression: gz
+                dir: arm/frc{year}-{name}
+                untar_dir: roborio-academic
+                url: https://github.com/wpilibsuite/opensdk/releases/download/v{year}-{build}/cortexa9_vfpv3-roborio-academic-{year}-x86_64-linux-gnu-Toolchain-{name}.tgz
+                targets:
+                  - year: 2023
+                    build: 4
+                    name: 12.1.0
           - type: tarballs
             arduino:
               check_exe: hardware/tools/avr/bin/avr-g++ --version

From 8096c251f588ea6659a913d75b5339a05922ab07 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 31 Oct 2022 21:51:57 -0500
Subject: [PATCH 031/161] Minor tweaks

---
 bin/yaml/cpp.yaml | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 251137c32..a276fc220 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -184,9 +184,17 @@ compilers:
               - gcc-arm-none-eabi-5_4-2016q3 # name unused in this case
           - type: tarballs
             frc:
-              - check_exe: roborio/bin/arm-frc{year}-linux-gnueabi-g++ --version
-                compression: gz
-                dir: arm/frc{year}-{name}
+              compression: gz
+              dir: arm/frc{year}-{name}
+              check_exe: bin/arm-frc{year}-linux-gnueabi-g++ --version
+              untar_dir: roborio-academic
+              url: https://github.com/wpilibsuite/opensdk/releases/download/v{year}-{build}/cortexa9_vfpv3-roborio-academic-{year}-x86_64-linux-gnu-Toolchain-{name}.tgz
+              targets:
+                - year: 2023
+                  build: 4
+                  name: 12.1.0
+              older:
+                check_exe: roborio/bin/arm-frc{year}-linux-gnueabi-g++ --version
                 untar_dir: frc{year}
                 url: https://github.com/wpilibsuite/toolchain-builder/releases/download/v{year}-{build}/FRC-{year}-Linux-Toolchain-{name}.tar.gz
                 targets:
@@ -197,15 +205,6 @@ compilers:
                     build: 1
                     name: 7.3.0
                     url: https://github.com/wpilibsuite/roborio-toolchain/releases/download/v{year}-{build}/FRC-{year}-Linux-Toolchain-{name}.tar.gz
-              - check_exe: bin/arm-frc{year}-linux-gnueabi-g++ --version
-                compression: gz
-                dir: arm/frc{year}-{name}
-                untar_dir: roborio-academic
-                url: https://github.com/wpilibsuite/opensdk/releases/download/v{year}-{build}/cortexa9_vfpv3-roborio-academic-{year}-x86_64-linux-gnu-Toolchain-{name}.tgz
-                targets:
-                  - year: 2023
-                    build: 4
-                    name: 12.1.0
           - type: tarballs
             arduino:
               check_exe: hardware/tools/avr/bin/avr-g++ --version

From 70e48db0b8219b365b018f5a1dda6d81742fc188 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 31 Oct 2022 19:52:47 -0700
Subject: [PATCH 032/161] add stdexec C++ library (#854)

---
 bin/yaml/libraries.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index 23d3708f8..3efdf893f 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -1287,6 +1287,13 @@ libraries:
         targets:
         - trunk
         type: github
+      stdexec:
+        check_file: include/stdexec/execution.hpp
+        method: nightlyclone
+        repo: NVIDIA/stdexec
+        targets:
+        - trunk
+        type: github
       taojson:
         build_type: none
         check_file: include/tao/json.hpp

From 07af06fbb3ae6ad3ba888b8fac7898b329038b84 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Mon, 31 Oct 2022 19:53:32 -0700
Subject: [PATCH 033/161] add nvhpc-sdk C++ compilers (#855)

---
 bin/yaml/cpp.yaml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index a276fc220..ff3c6ba30 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -713,6 +713,36 @@ compilers:
           fetch:
             - https://registrationcenter-download.intel.com/akdlm/irc_nas/18703/l_fortran-compiler_p_{name}_offline.sh install.sh
           script: *intel-one-install-script
+    nvhpc-sdk:
+      type: script
+      check_stderr_on_stdout: true
+      check_exe: hpc_sdk/Linux_{arch}/{version}/compilers/bin/nvc++ --version
+      targets:
+        - arch: "x86_64"
+          version: "22.7"
+          script: &nvhpc-sdk-install-script |
+            mkdir -p "nvhpc_sdk_{version}"
+            tar -f "nvhpc_sdk_{version}.tar.gz" -C "nvhpc_sdk_{version}" --wildcards --strip-components=1 -pzx "nvhpc_*/*"
+            rm -rf "nvhpc_sdk_{version}.tar.gz"
+            NVHPC_SILENT=true NVHPC_INSTALL_DIR=$CE_STAGING_DIR bash "nvhpc_sdk_{version}/install"
+            rm -rf "nvhpc_sdk_{version}"
+          fetch:
+            - https://developer.download.nvidia.com/hpc-sdk/22.7/nvhpc_2022_227_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
+        - arch: "aarch64"
+          version: "22.7"
+          script: *nvhpc-sdk-install-script
+          fetch:
+            - https://developer.download.nvidia.com/hpc-sdk/22.7/nvhpc_2022_227_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
+        - arch: "x86_64"
+          version: "22.9"
+          script: *nvhpc-sdk-install-script
+          fetch:
+            - https://developer.download.nvidia.com/hpc-sdk/22.9/nvhpc_2022_229_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
+        - arch: "aarch64"
+          version: "22.9"
+          script: *nvhpc-sdk-install-script
+          fetch:
+            - https://developer.download.nvidia.com/hpc-sdk/22.9/nvhpc_2022_229_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
     nightly:
       if: nightly
       gcc:

From 0376a2ff7b331f07d77c4ac180de9099da271743 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 31 Oct 2022 22:19:15 -0500
Subject: [PATCH 034/161] Fix %% in output

---
 bin/lib/installation_context.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/lib/installation_context.py b/bin/lib/installation_context.py
index 3595816f2..5bb7d9e6b 100644
--- a/bin/lib/installation_context.py
+++ b/bin/lib/installation_context.py
@@ -118,7 +118,7 @@ def fetch_to(self, url: str, fd: IO[bytes]) -> None:
             now = time.time()
             if now >= report_time:
                 if length != 0:
-                    _LOGGER.info("%.1f of %s...", 100.0 * fetched / length, url)
+                    _LOGGER.info("%.1f%% of %s...", 100.0 * fetched / length, url)
                 report_time = now + report_every_secs
         _LOGGER.info("100%% of %s", url)
         fd.flush()

From 67c5ccace3a832560013923393eca516356c88bd Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 31 Oct 2022 22:19:36 -0500
Subject: [PATCH 035/161] Fix up nvhpc-sdk (cc @trxcllnt)

---
 bin/yaml/cpp.yaml | 40 ++++++++++++++++++----------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index ff3c6ba30..14e6280b4 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -715,34 +715,30 @@ compilers:
           script: *intel-one-install-script
     nvhpc-sdk:
       type: script
+      dir: hpc_sdk/Linux_{arch}/{version}
       check_stderr_on_stdout: true
-      check_exe: hpc_sdk/Linux_{arch}/{version}/compilers/bin/nvc++ --version
+      check_exe: compilers/bin/nvc++ --version
+      fetch:
+        - https://developer.download.nvidia.com/hpc-sdk/{version}/nvhpc_2022_{{version|replace(".","")}}_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
+      script: |
+        mkdir -p "nvhpc_sdk_{version}"
+        tar -f "nvhpc_sdk_{version}.tar.gz" -C "nvhpc_sdk_{version}" --wildcards --strip-components=1 -pzx "nvhpc_*/*"
+        rm -rf "nvhpc_sdk_{version}.tar.gz"
+        NVHPC_SILENT=true NVHPC_INSTALL_DIR=$CE_STAGING_DIR/hpc_sdk bash "nvhpc_sdk_{version}/install"
+        rm -rf "nvhpc_sdk_{version}"
       targets:
-        - arch: "x86_64"
+        - name: x86_64_22_7
+          arch: x86_64
           version: "22.7"
-          script: &nvhpc-sdk-install-script |
-            mkdir -p "nvhpc_sdk_{version}"
-            tar -f "nvhpc_sdk_{version}.tar.gz" -C "nvhpc_sdk_{version}" --wildcards --strip-components=1 -pzx "nvhpc_*/*"
-            rm -rf "nvhpc_sdk_{version}.tar.gz"
-            NVHPC_SILENT=true NVHPC_INSTALL_DIR=$CE_STAGING_DIR bash "nvhpc_sdk_{version}/install"
-            rm -rf "nvhpc_sdk_{version}"
-          fetch:
-            - https://developer.download.nvidia.com/hpc-sdk/22.7/nvhpc_2022_227_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
-        - arch: "aarch64"
+        - name: aarch64_22_7
+          arch: aarch64
           version: "22.7"
-          script: *nvhpc-sdk-install-script
-          fetch:
-            - https://developer.download.nvidia.com/hpc-sdk/22.7/nvhpc_2022_227_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
-        - arch: "x86_64"
+        - name: x86_64_22_9
+          arch: x86_64
           version: "22.9"
-          script: *nvhpc-sdk-install-script
-          fetch:
-            - https://developer.download.nvidia.com/hpc-sdk/22.9/nvhpc_2022_229_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
-        - arch: "aarch64"
+        - name: aarch64_22_9
+          arch: aarch64
           version: "22.9"
-          script: *nvhpc-sdk-install-script
-          fetch:
-            - https://developer.download.nvidia.com/hpc-sdk/22.9/nvhpc_2022_229_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
     nightly:
       if: nightly
       gcc:

From b5c196e7c439f4348779897cf02d46c49b6621a2 Mon Sep 17 00:00:00 2001
From: Enrico Seiler <eseiler@users.noreply.github.com>
Date: Tue, 1 Nov 2022 04:20:38 +0100
Subject: [PATCH 036/161] Add SeqAn3 C++ library (#856)

---
 bin/yaml/libraries.yaml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index 3efdf893f..74506f0c6 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -1255,6 +1255,14 @@ libraries:
         targets:
         - trunk
         type: github
+      seqan3:
+        build_type: none
+        check_file: include/seqan3/version.hpp
+        method: nightlyclone
+        repo: seqan/seqan3
+        targets:
+        - trunk
+        type: github
       seastar:
         build_type: none
         check_file: README.md
@@ -1514,6 +1522,14 @@ libraries:
       targets:
       - seastar-18.08.0
       type: github
+    seqan3:
+      build_type: none
+      check_file: include/seqan3/version.hpp
+      method: clone_branch
+      repo: seqan/seqan3
+      targets:
+      - 3.2.0
+      type: github
     sol2:
       build_type: none
       check_file: include/sol/sol.hpp

From 620a1d017cd964747daf3aa19f2a857ace6a3a58 Mon Sep 17 00:00:00 2001
From: Quinton Miller <nicetas.c@gmail.com>
Date: Tue, 1 Nov 2022 11:20:56 +0800
Subject: [PATCH 037/161] Update Crystal 1.6.0 -> 1.6.1 (#858)

---
 bin/yaml/crystal.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/yaml/crystal.yaml b/bin/yaml/crystal.yaml
index 06dd86d13..ad3fe5543 100644
--- a/bin/yaml/crystal.yaml
+++ b/bin/yaml/crystal.yaml
@@ -15,7 +15,7 @@ compilers:
       - ln -s /opt/compiler-explorer/libs/libpcre/8.45/x86_64/lib/libpcre.a # for 1.1.1 and before
       - ln -s /opt/compiler-explorer/libs/libevent/2.1.12/x86_64/lib/libevent.a
     targets:
-      - 1.6.0
+      - 1.6.1
       - 1.5.1
       - 1.4.1
       - 1.3.2

From 8593cb87a75330cda0ece1c928254fd1991275ee Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 31 Oct 2022 22:36:30 -0500
Subject: [PATCH 038/161] Commenting out aarch64. CC @trxcllnt

---
 bin/yaml/cpp.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 14e6280b4..cf6f13ac3 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -730,15 +730,15 @@ compilers:
         - name: x86_64_22_7
           arch: x86_64
           version: "22.7"
-        - name: aarch64_22_7
-          arch: aarch64
-          version: "22.7"
+#        - name: aarch64_22_7
+#          arch: aarch64
+#          version: "22.7"
         - name: x86_64_22_9
           arch: x86_64
           version: "22.9"
-        - name: aarch64_22_9
-          arch: aarch64
-          version: "22.9"
+#        - name: aarch64_22_9
+#          arch: aarch64
+#          version: "22.9"
     nightly:
       if: nightly
       gcc:

From 720686a4fec23994085bd8764011e2706c492a4a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 31 Oct 2022 23:22:08 -0500
Subject: [PATCH 039/161] Check for origin/branch before using it, so we fix
 tags

---
 bin/lib/installable/git.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bin/lib/installable/git.py b/bin/lib/installable/git.py
index 9334760f5..a061bbd7e 100644
--- a/bin/lib/installable/git.py
+++ b/bin/lib/installable/git.py
@@ -27,7 +27,7 @@ def _git_raw(logger: logging.Logger, cwd: Path, *git_args: Union[str, Path]) ->
 
 
 def _remote_default_branch_for(logger: logging.Logger, git_repo: Path) -> str:
-    match_re = re.compile(r"^ref:\s+refs/heads/([^\s]+)\s+HEAD$")
+    match_re = re.compile(r"^ref:\s+refs/heads/(\S+)\s+HEAD$")
     for line in _git_raw(logger, git_repo, "ls-remote", "--symref", "origin", "HEAD").splitlines(keepends=False):
         if match := match_re.match(line):
             return match.group(1)
@@ -130,7 +130,10 @@ def _git(*git_args: Union[str, Path]) -> str:
         _git("clean", "-ffdx")
         _git("reset", "--hard", "HEAD")
 
-        _git("checkout", f"origin/{branch or self._find_remote_branch(dest)}")
+        to_checkout = branch or self._find_remote_branch(dest)
+        if _git("branch", "--list", f"origin/{to_checkout}").strip():
+            to_checkout = f"origin/{to_checkout}"
+        _git("checkout", to_checkout)
 
         _git("submodule", "sync")
         _git("submodule", "update", "--init", *self._update_args())

From 2c4d913d7f1a8b71694d75f804155378698be43a Mon Sep 17 00:00:00 2001
From: James Murphy <72399781+mCodingLLC@users.noreply.github.com>
Date: Tue, 1 Nov 2022 17:37:26 -0500
Subject: [PATCH 040/161] Add Python 3.11 (#860)

---
 bin/yaml/python.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/yaml/python.yaml b/bin/yaml/python.yaml
index 27a01a5d9..027a6f01d 100644
--- a/bin/yaml/python.yaml
+++ b/bin/yaml/python.yaml
@@ -9,3 +9,4 @@ compilers:
       - 3.8.1
       - 3.9.6
       - 3.10.0
+      - 3.11.0

From 9fa914109241d9614ba5b419ffe6773066d608a2 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 2 Nov 2022 08:37:40 -0500
Subject: [PATCH 041/161] Add python-builder to bespoke builds

---
 .github/workflows/bespoke-build.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/bespoke-build.yaml b/.github/workflows/bespoke-build.yaml
index b1bb68359..04d44c6cf 100644
--- a/.github/workflows/bespoke-build.yaml
+++ b/.github/workflows/bespoke-build.yaml
@@ -14,6 +14,7 @@ on:
         - clang
         - misc
         - ocaml
+        - python
       version:
         description: The version to build (or architecture and version for cross)
         required: true

From cd01f500b4f67aad0bf19c0e2b0cef7875cd7f76 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Wed, 2 Nov 2022 08:52:00 -0500
Subject: [PATCH 042/161] Ahem

---
 .github/workflows/bespoke-build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bespoke-build.yaml b/.github/workflows/bespoke-build.yaml
index 04d44c6cf..e278165e0 100644
--- a/.github/workflows/bespoke-build.yaml
+++ b/.github/workflows/bespoke-build.yaml
@@ -23,7 +23,7 @@ on:
         default: build.sh
         required: true
 jobs:
-  bepoke-build:
+  bespoke-build:
     runs-on: [ 'self-hosted', 'ce', 'ubuntu' ]
     steps:
       - name: Start from a clean directory

From bfa981a0ed5124f9962bc837b54d486251946491 Mon Sep 17 00:00:00 2001
From: Miguel Ojeda <ojeda@users.noreply.github.com>
Date: Fri, 4 Nov 2022 09:00:00 +0100
Subject: [PATCH 043/161] Rust 1.65.0 (#862)

Signed-off-by: Miguel Ojeda <ojeda@kernel.org>
---
 bin/yaml/rust.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/yaml/rust.yaml b/bin/yaml/rust.yaml
index 243957e85..ddf2d2e6c 100644
--- a/bin/yaml/rust.yaml
+++ b/bin/yaml/rust.yaml
@@ -77,6 +77,7 @@ compilers:
         - 1.62.0
         - 1.63.0
         - 1.64.0
+        - 1.65.0
       nightly:
         if: nightly
         targets:

From 756bc4dfce7c616c3249c7b8309c9a1910efb70c Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Fri, 4 Nov 2022 05:10:21 -0700
Subject: [PATCH 044/161] run makelocalrc with explicit paths to GCC toolchains
 (#859)

---
 bin/yaml/cpp.yaml | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index cf6f13ac3..0d1f84ee8 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -719,13 +719,33 @@ compilers:
       check_stderr_on_stdout: true
       check_exe: compilers/bin/nvc++ --version
       fetch:
-        - https://developer.download.nvidia.com/hpc-sdk/{version}/nvhpc_2022_{{version|replace(".","")}}_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}.tar.gz
+        - https://developer.download.nvidia.com/hpc-sdk/{version}/nvhpc_2022_{{version|replace(".","")}}_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}_{arch}.tar.gz
       script: |
-        mkdir -p "nvhpc_sdk_{version}"
-        tar -f "nvhpc_sdk_{version}.tar.gz" -C "nvhpc_sdk_{version}" --wildcards --strip-components=1 -pzx "nvhpc_*/*"
-        rm -rf "nvhpc_sdk_{version}.tar.gz"
-        NVHPC_SILENT=true NVHPC_INSTALL_DIR=$CE_STAGING_DIR/hpc_sdk bash "nvhpc_sdk_{version}/install"
-        rm -rf "nvhpc_sdk_{version}"
+        mkdir -p "nvhpc_sdk_{version}_{arch}";
+        tar -f "nvhpc_sdk_{version}_{arch}.tar.gz" -C "nvhpc_sdk_{version}_{arch}" --wildcards --strip-components=1 -pzx "nvhpc_*/*"
+        rm -rf "nvhpc_sdk_{version}_{arch}.tar.gz"
+
+        NVHPC_SILENT=true \
+        NVHPC_INSTALL_DIR=$CE_STAGING_DIR/hpc_sdk \
+          bash "nvhpc_sdk_{version}_{arch}/install"
+
+        rm -rf "nvhpc_sdk_{version}_{arch}"
+
+        if [[ "{arch}" == "x86_64" ]]; then
+          bash "$CE_STAGING_DIR/hpc_sdk/Linux_{arch}/{version}/compilers/bin/makelocalrc" \
+            -x "$CE_STAGING_DIR/hpc_sdk/Linux_{arch}/{version}/compilers/bin" \
+            -gcc /opt/compiler-explorer/gcc-12.2.0/bin/gcc \
+            -gpp /opt/compiler-explorer/gcc-12.2.0/bin/g++ \
+            -g77 /opt/compiler-explorer/gcc-12.2.0/bin/gfortran \
+            ;
+        elif [[ "{arch}" == "aarch64" ]]; then
+          bash "$CE_STAGING_DIR/hpc_sdk/Linux_{arch}/{version}/compilers/bin/makelocalrc" \
+            -x "$CE_STAGING_DIR/hpc_sdk/Linux_{arch}/{version}/compilers/bin" \
+            -gcc /opt/compiler-explorer/arm64/gcc-12.2.0/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc \
+            -gpp /opt/compiler-explorer/arm64/gcc-12.2.0/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-g++ \
+            -g77 /opt/compiler-explorer/arm64/gcc-12.2.0/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gfortran \
+            ;
+        fi
       targets:
         - name: x86_64_22_7
           arch: x86_64

From a57701ea736f8a5a6328e6428bf49e8bd91e1261 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Fri, 4 Nov 2022 19:36:21 +0100
Subject: [PATCH 045/161] draft (#863)

---
 init/start.sh        |   7 ++-
 packer-gpu-node.json |  73 ++++++++++++++++++++++++++++++
 setup-gpu-node.sh    | 104 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 183 insertions(+), 1 deletion(-)
 create mode 100644 packer-gpu-node.json
 create mode 100644 setup-gpu-node.sh

diff --git a/init/start.sh b/init/start.sh
index d4a4bf7d4..5cacef308 100755
--- a/init/start.sh
+++ b/init/start.sh
@@ -8,6 +8,7 @@ CE_USER=ce
 DEPLOY_DIR=${PWD}/.deploy
 COMPILERS_ARG=
 COMPILERS_FILE=$DEPLOY_DIR/discovered-compilers.json
+CE_PROP_ENV=amazon
 
 echo Running in environment "${ENV}"
 # shellcheck disable=SC1090
@@ -102,6 +103,10 @@ if [[ "${ENV}" == "runner" ]]; then
   exit
 fi
 
+if [[ "${ENV}" == "gpu" ]]; then
+  CE_PROP_ENV=gpu
+fi
+
 # shellcheck disable=SC2086
 exec sudo -u ${CE_USER} -H --preserve-env=NODE_ENV -- \
     /opt/node/bin/node \
@@ -110,7 +115,7 @@ exec sudo -u ${CE_USER} -H --preserve-env=NODE_ENV -- \
     --suppressConsoleLog \
     --logHost "${LOG_DEST_HOST}" \
     --logPort "${LOG_DEST_PORT}" \
-    --env amazon \
+    --env ${CE_PROP_ENV} \
     --port 10240 \
     --metricsPort 10241 \
     --loki "http://127.0.0.1:3500" \
diff --git a/packer-gpu-node.json b/packer-gpu-node.json
new file mode 100644
index 000000000..d5417829c
--- /dev/null
+++ b/packer-gpu-node.json
@@ -0,0 +1,73 @@
+{
+  "builders": [
+    {
+      "type": "amazon-ebs",
+      "region": "us-east-1",
+      "access_key": "{{user `MY_ACCESS_KEY`}}",
+      "secret_key": "{{user `MY_SECRET_KEY`}}",
+      "source_ami_filter": {
+          "filters": {
+              "virtualization-type": "hvm",
+              "name": "ubuntu/images/*ubuntu-focal-20.04-amd64-server-*",
+              "root-device-type": "ebs"
+          },
+          "owners": ["099720109477"],
+          "most_recent": true
+      },
+      "instance_type": "g4dn.xlarge",
+      "ssh_username": "ubuntu",
+      "ami_name": "compiler-explorer gpu packer 20.04 @ {{timestamp}}",
+      "tags": {"Site":  "CompilerExplorer"},
+      "run_volume_tags": {"Site":  "CompilerExplorer"},
+      "associate_public_ip_address": true,
+      "iam_instance_profile": "XaniaBlog",
+      "vpc_id": "vpc-17209172",
+      "security_group_id": "sg-f53f9f80",
+      "subnet_id": "subnet-1df1e135",
+      "ami_block_device_mappings": [
+        {
+          "device_name": "/dev/sda1",
+          "volume_size": 6,
+          "delete_on_termination": true,
+          "volume_type": "gp2"
+        }
+      ],
+      "launch_block_device_mappings": [
+        {
+          "device_name": "/dev/sda1",
+          "volume_size": 20,
+          "delete_on_termination": true,
+          "volume_type": "gp2"
+        }
+      ]
+    }
+  ],
+
+  "variables": {
+    "BRANCH": "main",
+    "MY_ACCESS_KEY": "",
+    "MY_SECRET_KEY": ""
+  },
+
+  "provisioners": [
+    { "type": "file", "source": "packer", "destination": "/home/ubuntu/" },
+    {
+      "type": "shell",
+      "execute_command" : "{{ .Vars }} sudo -E bash '{{ .Path }}'",
+      "inline": [
+        "set -euo pipefail",
+        "while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done",
+        "export DEBIAN_FRONTEND=noninteractive",
+        "mkdir -p /root/.ssh",
+        "cp /home/ubuntu/packer/known_hosts /root/.ssh/",
+        "cp /home/ubuntu/packer/known_hosts /home/ubuntu/.ssh/",
+        "rm -rf /home/ubuntu/packer",
+        "apt-get -y update",
+        "apt-get -y install git",
+        "git clone -b {{user `BRANCH`}} https://github.com/compiler-explorer/infra.git /infra",
+        "cd /infra",
+        "env PACKER_SETUP=yes bash setup-gpu-node.sh 2>&1 | tee /tmp/setup.log"
+      ]
+    }
+  ]
+}
diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
new file mode 100644
index 000000000..4151b1d73
--- /dev/null
+++ b/setup-gpu-node.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -exuo pipefail
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+env EXTRA_NFS_ARGS=",ro" "${DIR}/setup-common.sh"
+
+apt-get -y update
+apt-get -y install software-properties-common
+dpkg --add-architecture i386
+curl -s https://dl.winehq.org/wine-builds/winehq.key | apt-key add -
+# TODO at some point see if we can upgrade wine
+apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ bionic main'
+apt-get install -y \
+    binutils-multiarch \
+    bison \
+    bzip2 \
+    cgroup-tools \
+    curl \
+    dos2unix \
+    file \
+    flex \
+    g++ \
+    gawk \
+    gcc \
+    gnat \
+    jq \
+    libapparmor-dev \
+    libc6-dev-i386 \
+    libdatetime-perl \
+    libelf-dev \
+    libgmp3-dev \
+    libprotobuf-dev \
+    libnl-route-3-dev \
+    libwww-perl \
+    linux-libc-dev \
+    make \
+    nfs-common \
+    nginx \
+    patch \
+    pkg-config \
+    protobuf-compiler \
+    python3-pip \
+    python3.8-venv \
+    s3cmd \
+    subversion \
+    texinfo \
+    unzip \
+    wget \
+    winehq-stable=4.0.3~bionic \
+    wine-stable=4.0.3~bionic \
+    wine-stable-amd64=4.0.3~bionic \
+    wine-stable-i386=4.0.3~bionic \
+    xz-utils
+
+pushd /tmp
+git clone https://github.com/apmorton/firejail.git firejail-apmorton
+cd firejail-apmorton
+git checkout 0.9.58.2-ce-patch.1
+./configure --enable-apparmor --prefix /usr/local/firejail-0.9.58.2-ce-patch.1
+make "-j$(nproc)"
+make install
+popd
+
+ln -s /usr/local/firejail-0.9.58.2-ce-patch.1/bin/firejail /usr/local/bin
+
+pushd /tmp
+git clone https://github.com/netblue30/firejail.git
+cd firejail
+git checkout 0.9.70
+./configure --enable-apparmor --prefix /usr/local/firejail-0.9.70
+make "-j$(nproc)"
+make install
+popd
+
+pushd /tmp
+git clone --recursive --branch 3.0 https://github.com/google/nsjail.git
+cd nsjail
+make "-j$(nproc)"
+cp nsjail /usr/local/bin/nsjail
+popd
+
+
+pushd /opt
+# node.js
+TARGET_NODE_VERSION=v16.17.1
+echo "Installing node ${TARGET_NODE_VERSION}"
+curl -sL "https://nodejs.org/dist/${TARGET_NODE_VERSION}/node-${TARGET_NODE_VERSION}-linux-x64.tar.gz" | tar zxf - && mv node-${TARGET_NODE_VERSION}-linux-x64 node
+popd
+
+cp nginx/nginx.conf /etc/nginx/nginx.conf
+systemctl restart nginx
+
+cp /infra/init/compiler-explorer.service /lib/systemd/system/compiler-explorer.service
+systemctl daemon-reload
+systemctl enable compiler-explorer
+
+adduser --system --group ce
+
+# setup nvidia drivers https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-nouveau-ubuntu
+
+echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
+update-initramfs -u

From 306d59839010127a20e4a847fc39ac7dd6a47db7 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 19:38:13 +0100
Subject: [PATCH 046/161] add make

---
 Makefile | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Makefile b/Makefile
index 24290ca96..c3f8cfe4e 100644
--- a/Makefile
+++ b/Makefile
@@ -36,6 +36,10 @@ packer-admin: config.json  ## Builds the base image for the admin server
 packer-conan: config.json  ## Builds the base image for the CE conan-server
 	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer-conan.json
 
+.PHONY: packer-gpu-node
+packer-gpu-node: config.json  ## Builds the base image for the CE conan-server
+	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer-gpu-node.json
+
 .PHONY: packer-win
 packer-win: config.json  ## Builds the base image for the CE windows
 	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer-win.json

From d06705cfd4429c675d3e8d5a2f2ce5c358a36b06 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 19:43:04 +0100
Subject: [PATCH 047/161] comment

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c3f8cfe4e..7af84be70 100644
--- a/Makefile
+++ b/Makefile
@@ -37,7 +37,7 @@ packer-conan: config.json  ## Builds the base image for the CE conan-server
 	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer-conan.json
 
 .PHONY: packer-gpu-node
-packer-gpu-node: config.json  ## Builds the base image for the CE conan-server
+packer-gpu-node: config.json  ## Builds the base image for the CE gpu nodes
 	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer-gpu-node.json
 
 .PHONY: packer-win

From 163166afb635dda5efaebfa96003273a756be30a Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 20:45:11 +0100
Subject: [PATCH 048/161] attempt to add gpu node

---
 terraform/ec2.tf | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)

diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index 522b45be1..0ba115908 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,6 +2,7 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
+  gpu_image_id     = "ami-0d5f3ae70c826d948"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 
@@ -129,3 +130,42 @@ resource "aws_instance" "CERunner" {
     Name = "CERunner"
   }
 }
+
+resource "aws_instance" "GPUNode" {
+  ami                         = local.gpu_image_id
+  iam_instance_profile        = aws_iam_instance_profile.CompilerExplorerRole.name
+  ebs_optimized               = false
+  instance_type               = "g4dn.xlarge"
+  monitoring                  = false
+  key_name                    = "mattgodbolt"
+  subnet_id                   = local.admin_subnet
+  vpc_security_group_ids      = [aws_security_group.CompilerExplorer.id]
+  associate_public_ip_address = true
+  source_dest_check           = false
+  user_data                   = "gpu"
+
+  lifecycle {
+    ignore_changes = [
+      // Seemingly needed to not replace stopped instances
+      associate_public_ip_address
+    ]
+  }
+
+  tag_specifications {
+    resource_type = "volume"
+
+    tags = {
+      Site = "CompilerExplorer"
+    }
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+
+    tags = {
+      Site        = "CompilerExplorer"
+      Environment = "GPU"
+      Name        = "GPUNode"
+    }
+  }
+}

From c7da1ae803f8b9153b4153e8d19f52733d4294d0 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 20:49:07 +0100
Subject: [PATCH 049/161] fix

---
 terraform/ec2.tf | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index 0ba115908..516e36954 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -151,21 +151,14 @@ resource "aws_instance" "GPUNode" {
     ]
   }
 
-  tag_specifications {
-    resource_type = "volume"
-
-    tags = {
-      Site = "CompilerExplorer"
-    }
+  root_block_device {
+    volume_type           = "gp2"
+    volume_size           = 24
+    delete_on_termination = false
   }
 
-  tag_specifications {
-    resource_type = "instance"
-
-    tags = {
-      Site        = "CompilerExplorer"
-      Environment = "GPU"
-      Name        = "GPUNode"
-    }
+  tags = {
+    Site        = "CompilerExplorer"
+    Name        = "GPUNode"
   }
 }

From a5b96914b45531090f91f52968cfcfc04434f589 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 20:55:20 +0100
Subject: [PATCH 050/161] oops

---
 site-gpu.sh | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 site-gpu.sh

diff --git a/site-gpu.sh b/site-gpu.sh
new file mode 100644
index 000000000..6c22434f5
--- /dev/null
+++ b/site-gpu.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Sourced when running in gpu mode
+export BRANCH=gpu
+export EXTRA_ARGS='--env staging --ensureNoIdClash'
+export NODE_ENV=production

From 942f2cf31c8937a9e74c2998d213909f3660dcdf Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Fri, 4 Nov 2022 21:16:40 +0100
Subject: [PATCH 051/161] Update site-staging.sh

---
 site-staging.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/site-staging.sh b/site-staging.sh
index 5ba0e0372..3aa86efa7 100644
--- a/site-staging.sh
+++ b/site-staging.sh
@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
 # Sourced when running in staging mode
 export BRANCH=staging
-export EXTRA_ARGS='--env staging --ensureNoIdClash'
 export NODE_ENV=production

From 619f5103cbcf367a8db4268efd64255bcd9972a5 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 21:31:49 +0100
Subject: [PATCH 052/161] change node installer url

---
 setup-gpu-node.sh | 2 +-
 setup-node.sh     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index 4151b1d73..e0e7c5b0e 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -86,7 +86,7 @@ pushd /opt
 # node.js
 TARGET_NODE_VERSION=v16.17.1
 echo "Installing node ${TARGET_NODE_VERSION}"
-curl -sL "https://nodejs.org/dist/${TARGET_NODE_VERSION}/node-${TARGET_NODE_VERSION}-linux-x64.tar.gz" | tar zxf - && mv node-${TARGET_NODE_VERSION}-linux-x64 node
+curl -sL "https://nodejs.org/dist/${TARGET_NODE_VERSION}/node-${TARGET_NODE_VERSION}-linux-x64.tar.xz" | tar xJf - && mv node-${TARGET_NODE_VERSION}-linux-x64 node
 popd
 
 cp nginx/nginx.conf /etc/nginx/nginx.conf
diff --git a/setup-node.sh b/setup-node.sh
index e478de32f..1a6913f71 100644
--- a/setup-node.sh
+++ b/setup-node.sh
@@ -86,7 +86,7 @@ pushd /opt
 # node.js
 TARGET_NODE_VERSION=v16.17.1
 echo "Installing node ${TARGET_NODE_VERSION}"
-curl -sL "https://nodejs.org/dist/${TARGET_NODE_VERSION}/node-${TARGET_NODE_VERSION}-linux-x64.tar.gz" | tar zxf - && mv node-${TARGET_NODE_VERSION}-linux-x64 node
+curl -sL "https://nodejs.org/dist/${TARGET_NODE_VERSION}/node-${TARGET_NODE_VERSION}-linux-x64.tar.xz" | tar xJf - && mv node-${TARGET_NODE_VERSION}-linux-x64 node
 popd
 
 cp nginx/nginx.conf /etc/nginx/nginx.conf

From fd9e8108a3897c2397152b3aad67d4fabc7bcc36 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 21:44:57 +0100
Subject: [PATCH 053/161] new ami

---
 terraform/ec2.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index 516e36954..abf123dae 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,7 +2,7 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
-  gpu_image_id     = "ami-0d5f3ae70c826d948"
+  gpu_image_id     = "ami-0189800d9c1696715"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 

From 2d6d0e12a5e795988c4f01b6fd4f745c755594a5 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 22:42:12 +0100
Subject: [PATCH 054/161] gpu things

---
 terraform/cloudfront_gpu.tf | 86 +++++++++++++++++++++++++++++++++++++
 terraform/tg.tf             | 25 +++++++++++
 2 files changed, 111 insertions(+)
 create mode 100644 terraform/cloudfront_gpu.tf

diff --git a/terraform/cloudfront_gpu.tf b/terraform/cloudfront_gpu.tf
new file mode 100644
index 000000000..ebedf082d
--- /dev/null
+++ b/terraform/cloudfront_gpu.tf
@@ -0,0 +1,86 @@
+resource "aws_cloudfront_distribution" "gpu-compiler-explorer-com" {
+  origin {
+    domain_name = "compiler-explorer.s3.amazonaws.com"
+    origin_id   = "S3-compiler-explorer"
+  }
+  origin {
+    domain_name = aws_alb.GccExplorerApp.dns_name
+    origin_id   = "GccExplorerApp"
+    custom_origin_config {
+      http_port                = 1080
+      https_port               = 1443
+      origin_read_timeout      = 60
+      origin_keepalive_timeout = 60
+      origin_protocol_policy   = "https-only"
+      origin_ssl_protocols     = [
+        "TLSv1",
+        "TLSv1.2",
+        "TLSv1.1"
+      ]
+    }
+  }
+
+  enabled          = true
+  is_ipv6_enabled  = true
+  retain_on_delete = true
+  aliases          = [
+    "gpu.compiler-explorer.com"
+  ]
+
+  viewer_certificate {
+    acm_certificate_arn      = data.aws_acm_certificate.godbolt-org-et-al.arn
+    ssl_support_method       = "sni-only"
+    minimum_protocol_version = "TLSv1.1_2016"
+  }
+
+  logging_config {
+    include_cookies = false
+    bucket          = "compiler-explorer-logs.s3.amazonaws.com"
+    prefix          = "cloudfront/"
+  }
+
+  http_version = "http2"
+
+  restrictions {
+    geo_restriction {
+      restriction_type = "blacklist"
+      locations        = [
+        "CU",
+        "IR",
+        "KP",
+        "SD",
+        "SY"
+      ]
+    }
+  }
+
+  default_cache_behavior {
+    allowed_methods = [
+      "HEAD",
+      "DELETE",
+      "POST",
+      "GET",
+      "OPTIONS",
+      "PUT",
+      "PATCH"
+    ]
+    cached_methods = [
+      "HEAD",
+      "GET"
+    ]
+    forwarded_values {
+      cookies {
+        forward = "all"
+      }
+      query_string = true
+      headers      = [
+        "Accept",
+        "Host",
+        "Authorization"
+      ]
+    }
+    target_origin_id       = "GccExplorerApp"
+    viewer_protocol_policy = "redirect-to-https"
+    compress               = true
+  }
+}
diff --git a/terraform/tg.tf b/terraform/tg.tf
index 01bf96680..75b0f7de4 100644
--- a/terraform/tg.tf
+++ b/terraform/tg.tf
@@ -55,3 +55,28 @@ resource "aws_alb_target_group_attachment" "CEConanServerTargetInstance" {
   target_id        = aws_instance.ConanNode.id
   port             = 1080
 }
+
+resource "aws_alb_target_group" "gpu" {
+  lifecycle {
+    create_before_destroy = true
+  }
+  name                 = "GPUGroup"
+  port                 = 1080
+  protocol             = "HTTP"
+  vpc_id               = module.ce_network.vpc.id
+  deregistration_delay = 15
+  health_check {
+    path                = "/healthcheck"
+    timeout             = 3
+    unhealthy_threshold = 3
+    healthy_threshold   = 2
+    interval            = 5
+    protocol            = "HTTP"
+  }
+}
+
+resource "aws_alb_target_group_attachment" "GPUServerTargetInstance" {
+  target_group_arn = aws_alb_target_group.gpu.id
+  target_id        = aws_instance.GPUNode.id
+  port             = 1080
+}

From 704598a9030c3e7e96b08cdbfb2739424ea74425 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 23:05:32 +0100
Subject: [PATCH 055/161] i have no idea what im doing

---
 terraform/alb.tf            | 23 +++++++++++++++++++++++
 terraform/cloudfront_gpu.tf |  4 ++--
 terraform/security.tf       | 31 +++++++++++++++++++++++++++++++
 terraform/tg.tf             |  4 ++--
 4 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/terraform/alb.tf b/terraform/alb.tf
index 308a1c371..6340e194d 100644
--- a/terraform/alb.tf
+++ b/terraform/alb.tf
@@ -168,3 +168,26 @@ resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-https-stats" {
   }
   listener_arn = aws_alb_listener.compiler-explorer-alb-listen-https.arn
 }
+
+resource "aws_alb_listener" "gpu-alb-listen-http" {
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_alb_target_group.gpu.arn
+  }
+
+  load_balancer_arn = aws_alb.GccExplorerApp.arn
+  port              = 1081
+  protocol          = "HTTP"
+}
+
+resource "aws_alb_listener" "gpu-alb-listen-https" {
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_alb_target_group.gpu.arn
+  }
+  load_balancer_arn = aws_alb.GccExplorerApp.arn
+  port              = 1444
+  protocol          = "HTTPS"
+  ssl_policy        = "ELBSecurityPolicy-2015-05"
+  certificate_arn   = data.aws_acm_certificate.godbolt-org-et-al.arn
+}
diff --git a/terraform/cloudfront_gpu.tf b/terraform/cloudfront_gpu.tf
index ebedf082d..4ad8712f8 100644
--- a/terraform/cloudfront_gpu.tf
+++ b/terraform/cloudfront_gpu.tf
@@ -7,8 +7,8 @@ resource "aws_cloudfront_distribution" "gpu-compiler-explorer-com" {
     domain_name = aws_alb.GccExplorerApp.dns_name
     origin_id   = "GccExplorerApp"
     custom_origin_config {
-      http_port                = 1080
-      https_port               = 1443
+      http_port                = 1081
+      https_port               = 1444
       origin_read_timeout      = 60
       origin_keepalive_timeout = 60
       origin_protocol_policy   = "https-only"
diff --git a/terraform/security.tf b/terraform/security.tf
index 1e29f6cae..7196160f7 100644
--- a/terraform/security.tf
+++ b/terraform/security.tf
@@ -52,6 +52,16 @@ resource "aws_security_group_rule" "CE_ConanHttpFromAlb" {
   description              = "Allow HTTP access from the ALB"
 }
 
+resource "aws_security_group_rule" "CE_GPUHttpFromAlb" {
+  security_group_id        = aws_security_group.CompilerExplorer.id
+  type                     = "ingress"
+  from_port                = 1081
+  to_port                  = 1081
+  source_security_group_id = aws_security_group.CompilerExplorerAlb.id
+  protocol                 = "tcp"
+  description              = "Allow HTTP access from the ALB"
+}
+
 resource "aws_security_group_rule" "CE_HttpsFromAlb" {
   security_group_id        = aws_security_group.CompilerExplorer.id
   type                     = "ingress"
@@ -72,6 +82,16 @@ resource "aws_security_group_rule" "CE_ConanHttpsFromAlb" {
   description              = "Allow HTTPS access from the ALB"
 }
 
+resource "aws_security_group_rule" "CE_GPUHttpsFromAlb" {
+  security_group_id        = aws_security_group.CompilerExplorer.id
+  type                     = "ingress"
+  from_port                = 1444
+  to_port                  = 1444
+  source_security_group_id = aws_security_group.CompilerExplorerAlb.id
+  protocol                 = "tcp"
+  description              = "Allow HTTPS access from the ALB"
+}
+
 resource "aws_security_group" "CompilerExplorerAlb" {
   vpc_id      = module.ce_network.vpc.id
   name        = "ce-alb-sg"
@@ -103,6 +123,17 @@ resource "aws_security_group_rule" "ALB_ConanHttpsFromAnywhere" {
   description       = "Allow HTTPS access from anywhere"
 }
 
+resource "aws_security_group_rule" "ALB_GPUHttpsFromAnywhere" {
+  security_group_id = aws_security_group.CompilerExplorerAlb.id
+  type              = "ingress"
+  from_port         = 1444
+  to_port           = 1444
+  cidr_blocks       = ["0.0.0.0/0"]
+  ipv6_cidr_blocks  = ["::/0"]
+  protocol          = "tcp"
+  description       = "Allow HTTPS access from anywhere"
+}
+
 resource "aws_security_group_rule" "ALB_EgressToAnywhere" {
   security_group_id = aws_security_group.CompilerExplorerAlb.id
   type              = "egress"
diff --git a/terraform/tg.tf b/terraform/tg.tf
index 75b0f7de4..527d483fd 100644
--- a/terraform/tg.tf
+++ b/terraform/tg.tf
@@ -61,7 +61,7 @@ resource "aws_alb_target_group" "gpu" {
     create_before_destroy = true
   }
   name                 = "GPUGroup"
-  port                 = 1080
+  port                 = 1081
   protocol             = "HTTP"
   vpc_id               = module.ce_network.vpc.id
   deregistration_delay = 15
@@ -78,5 +78,5 @@ resource "aws_alb_target_group" "gpu" {
 resource "aws_alb_target_group_attachment" "GPUServerTargetInstance" {
   target_group_arn = aws_alb_target_group.gpu.id
   target_id        = aws_instance.GPUNode.id
-  port             = 1080
+  port             = 1081
 }

From d82e1dfbac926dd8eb41e3ec5c8ff6ebb5d93812 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 23:34:38 +0100
Subject: [PATCH 056/161] rename

---
 terraform/tg.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/tg.tf b/terraform/tg.tf
index 527d483fd..aa6dca6c5 100644
--- a/terraform/tg.tf
+++ b/terraform/tg.tf
@@ -75,7 +75,7 @@ resource "aws_alb_target_group" "gpu" {
   }
 }
 
-resource "aws_alb_target_group_attachment" "GPUServerTargetInstance" {
+resource "aws_alb_target_group_attachment" "CEGPUServerTargetInstance" {
   target_group_arn = aws_alb_target_group.gpu.id
   target_id        = aws_instance.GPUNode.id
   port             = 1081

From 4b71d783ceb63ab72164790b6a97d5439648144c Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 23:38:42 +0100
Subject: [PATCH 057/161] use different port for gpu instance

---
 nginx/nginx-gpu.conf | 67 ++++++++++++++++++++++++++++++++++++++++++++
 setup-gpu-node.sh    |  2 +-
 2 files changed, 68 insertions(+), 1 deletion(-)
 create mode 100644 nginx/nginx-gpu.conf

diff --git a/nginx/nginx-gpu.conf b/nginx/nginx-gpu.conf
new file mode 100644
index 000000000..d7faa2765
--- /dev/null
+++ b/nginx/nginx-gpu.conf
@@ -0,0 +1,67 @@
+user www-data;
+worker_processes 1;
+pid /run/nginx.pid;
+
+events {
+	worker_connections 768;
+	# multi_accept on;
+}
+
+http {
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	##
+	# SSL Settings
+	##
+
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+	ssl_prefer_server_ciphers on;
+
+	##
+	# Logging Settings
+	##
+
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+	##
+	# Gzip Settings
+	##
+
+	gzip on;
+	gzip_disable "msie6";
+    gzip_proxied any;
+    gzip_http_version 1.1;
+    gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+    set_real_ip_from 172.30.0.0/16;
+    real_ip_header X-Forwarded-For;
+    client_max_body_size 10m;
+
+    server {
+        index index.html;
+        access_log /var/log/nginx/all.log;
+        error_log /var/log/nginx/all.err;
+		listen 1081;
+
+        location / {
+            proxy_pass http://localhost:10240;
+            proxy_set_header Host $host;
+
+            # uncomment the following line if you run your own instance
+            # proxy_set_header X-Forwarded-Proto $scheme;
+
+            # See https://ma.ttias.be/nginx-proxy-upstream-sent-big-header-reading-response-header-upstream/
+            # and https://github.com/mattgodbolt/compiler-explorer/issues/1423
+            proxy_buffer_size          128k;
+            proxy_buffers              4 256k;
+            proxy_busy_buffers_size    256k;
+        }
+    }
+}
diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index e0e7c5b0e..b317037a6 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -89,7 +89,7 @@ echo "Installing node ${TARGET_NODE_VERSION}"
 curl -sL "https://nodejs.org/dist/${TARGET_NODE_VERSION}/node-${TARGET_NODE_VERSION}-linux-x64.tar.xz" | tar xJf - && mv node-${TARGET_NODE_VERSION}-linux-x64 node
 popd
 
-cp nginx/nginx.conf /etc/nginx/nginx.conf
+cp nginx/nginx-gpu.conf /etc/nginx/nginx.conf
 systemctl restart nginx
 
 cp /infra/init/compiler-explorer.service /lib/systemd/system/compiler-explorer.service

From 8b9af680821b8f9b2d14d6c6ccd1afd7748a69b9 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 4 Nov 2022 23:46:36 +0100
Subject: [PATCH 058/161] new ami

---
 terraform/ec2.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index abf123dae..23a8e19f6 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,7 +2,7 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
-  gpu_image_id     = "ami-0189800d9c1696715"
+  gpu_image_id     = "ami-01b0af2e89e97e941"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 

From 9f6119fac49776894177caaf39217275cb80ee91 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 00:43:09 +0100
Subject: [PATCH 059/161] install nvidia driver

---
 setup-gpu-node.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index b317037a6..5ed35cc82 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -100,5 +100,10 @@ adduser --system --group ce
 
 # setup nvidia drivers https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-nouveau-ubuntu
 
+pushd /tmp
+wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
+sh cuda_11.8.0_520.61.05_linux.run --silent --driver
+popd
+
 echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
 update-initramfs -u

From 3f54db175ecf60b4bf0d0a0c9229b4c171291bbc Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 00:58:11 +0100
Subject: [PATCH 060/161] fix env

---
 site-gpu.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/site-gpu.sh b/site-gpu.sh
index 6c22434f5..0e4014693 100644
--- a/site-gpu.sh
+++ b/site-gpu.sh
@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
 # Sourced when running in gpu mode
 export BRANCH=gpu
-export EXTRA_ARGS='--env staging --ensureNoIdClash'
 export NODE_ENV=production

From b81916cf3f1222dbb71d6b67e0734929c4844c7f Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 01:10:56 +0100
Subject: [PATCH 061/161] new ami

---
 terraform/ec2.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index 23a8e19f6..97737a049 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,7 +2,7 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
-  gpu_image_id     = "ami-01b0af2e89e97e941"
+  gpu_image_id     = "ami-0e81e8732857a38c6"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 

From e99cd72c19ad57c4a70269bad51389708ef32d3a Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 13:51:27 +0100
Subject: [PATCH 062/161] new dbtable

---
 terraform/dynamodb.tf | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/terraform/dynamodb.tf b/terraform/dynamodb.tf
index 49661debd..ab99a5d0d 100644
--- a/terraform/dynamodb.tf
+++ b/terraform/dynamodb.tf
@@ -27,6 +27,35 @@ resource "aws_dynamodb_table" "links" {
   }
 }
 
+resource "aws_dynamodb_table" "gpulinks" {
+  name = "gpulinks"
+  lifecycle {
+    ignore_changes = [
+      read_capacity,
+      write_capacity
+    ]
+    prevent_destroy = true
+  }
+  billing_mode   = "PAY_PER_REQUEST"
+  read_capacity  = 1
+  write_capacity = 1
+  hash_key       = "prefix"
+  range_key      = "unique_subhash"
+
+  attribute {
+    name = "prefix"
+    type = "S"
+  }
+  attribute {
+    name = "unique_subhash"
+    type = "S"
+  }
+
+  point_in_time_recovery {
+    enabled = true
+  }
+}
+
 resource "aws_dynamodb_table" "versionslog" {
   name = "versionslog"
   lifecycle {

From 458634906c88f2e40fd570ca20728940c0072467 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 13:52:51 +0100
Subject: [PATCH 063/161] errorfix

---
 terraform/dynamodb.tf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/terraform/dynamodb.tf b/terraform/dynamodb.tf
index ab99a5d0d..5793ab290 100644
--- a/terraform/dynamodb.tf
+++ b/terraform/dynamodb.tf
@@ -37,8 +37,6 @@ resource "aws_dynamodb_table" "gpulinks" {
     prevent_destroy = true
   }
   billing_mode   = "PAY_PER_REQUEST"
-  read_capacity  = 1
-  write_capacity = 1
   hash_key       = "prefix"
   range_key      = "unique_subhash"
 

From f07aeb6a2b1c5b866222558de50d9ad460d52053 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Sat, 5 Nov 2022 18:40:01 +0100
Subject: [PATCH 064/161] fix devices

---
 setup-gpu-node.sh | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index 5ed35cc82..54e49d83c 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -107,3 +107,38 @@ popd
 
 echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
 update-initramfs -u
+
+# script from https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-verifications
+
+/sbin/modprobe nvidia
+
+if [ "$?" -eq 0 ]; then
+  # Count the number of NVIDIA controllers found.
+  NVDEVS=`lspci | grep -i NVIDIA`
+  N3D=`echo "$NVDEVS" | grep "3D controller" | wc -l`
+  NVGA=`echo "$NVDEVS" | grep "VGA compatible controller" | wc -l`
+
+  N=`expr $N3D + $NVGA - 1`
+  for i in `seq 0 $N`; do
+    mknod -m 666 /dev/nvidia$i c 195 $i
+  done
+
+  mknod -m 666 /dev/nvidiactl c 195 255
+
+else
+  exit 1
+fi
+
+/sbin/modprobe nvidia-uvm
+
+if [ "$?" -eq 0 ]; then
+  # Find out the major device number used by the nvidia-uvm driver
+  D=`grep nvidia-uvm /proc/devices | awk '{print $1}'`
+
+  mknod -m 666 /dev/nvidia-uvm c $D 0
+else
+  exit 1
+fi
+
+# /dev/nvidia-modeset
+/bin/nvidia-modprobe -m

From 639afc786531fc79e15c6e0320bfe5d1c2dfc5e5 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 21:23:58 +0100
Subject: [PATCH 065/161] attempt at rewrite script to pass shellcheck

---
 setup-gpu-node-devices.sh | 26 ++++++++++++++++++++++++++
 setup-gpu-node.sh         | 31 +------------------------------
 2 files changed, 27 insertions(+), 30 deletions(-)
 create mode 100755 setup-gpu-node-devices.sh

diff --git a/setup-gpu-node-devices.sh b/setup-gpu-node-devices.sh
new file mode 100755
index 000000000..a5fecbe3d
--- /dev/null
+++ b/setup-gpu-node-devices.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+if /sbin/modprobe nvidia; then
+  # Count the number of NVIDIA controllers found.
+  NVDEVS=$(lspci | grep -i NVIDIA)
+  N3D=$(echo "$NVDEVS" | grep -c "3D controller")
+  NVGA=$(echo "$NVDEVS" | grep -c "VGA compatible controller")
+
+  N=$((N3D + NVGA - 1))
+  for i in $(seq 0 $N); do
+    mknod -m 666 "/dev/nvidia$i" c 195 "$i"
+  done
+
+  mknod -m 666 /dev/nvidiactl c 195 255
+else
+  exit 1
+fi
+
+if /sbin/modprobe nvidia-uvm; then
+  # Find out the major device number used by the nvidia-uvm driver
+  D=$(grep nvidia-uvm /proc/devices | awk '{print $1}')
+
+  mknod -m 666 /dev/nvidia-uvm c "$D" 0
+else
+  exit 1
+fi
diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index 54e49d83c..1dff54e4e 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -109,36 +109,7 @@ echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/black
 update-initramfs -u
 
 # script from https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-verifications
-
-/sbin/modprobe nvidia
-
-if [ "$?" -eq 0 ]; then
-  # Count the number of NVIDIA controllers found.
-  NVDEVS=`lspci | grep -i NVIDIA`
-  N3D=`echo "$NVDEVS" | grep "3D controller" | wc -l`
-  NVGA=`echo "$NVDEVS" | grep "VGA compatible controller" | wc -l`
-
-  N=`expr $N3D + $NVGA - 1`
-  for i in `seq 0 $N`; do
-    mknod -m 666 /dev/nvidia$i c 195 $i
-  done
-
-  mknod -m 666 /dev/nvidiactl c 195 255
-
-else
-  exit 1
-fi
-
-/sbin/modprobe nvidia-uvm
-
-if [ "$?" -eq 0 ]; then
-  # Find out the major device number used by the nvidia-uvm driver
-  D=`grep nvidia-uvm /proc/devices | awk '{print $1}'`
-
-  mknod -m 666 /dev/nvidia-uvm c $D 0
-else
-  exit 1
-fi
+./setup-gpu-node-devices.sh
 
 # /dev/nvidia-modeset
 /bin/nvidia-modprobe -m

From 1662a55a2671b58b68b7444cfd05f1e289313ae0 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 21:34:55 +0100
Subject: [PATCH 066/161] new ami

---
 terraform/ec2.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index 97737a049..a82326b36 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,7 +2,7 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
-  gpu_image_id     = "ami-0e81e8732857a38c6"
+  gpu_image_id     = "ami-06b394f7def740c02"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 

From ef720bcf992625dc767f18958123ccece96ee84c Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 22:02:02 +0100
Subject: [PATCH 067/161] policy things

---
 terraform/security.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/security.tf b/terraform/security.tf
index 7196160f7..7c06efb4c 100644
--- a/terraform/security.tf
+++ b/terraform/security.tf
@@ -250,7 +250,7 @@ data "aws_iam_policy_document" "CeModifyStoredState" {
       "dynamodb:Scan",
       "dynamodb:Query"
     ]
-    resources = [aws_dynamodb_table.links.arn]
+    resources = [aws_dynamodb_table.links.arn,aws_dynamodb_table.gpulinks.arn]
   }
   statement {
     sid       = "S3AccessSid"

From 9093e7cce00e111c22ebe9a6ccaf2b5a272de147 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 22:26:10 +0100
Subject: [PATCH 068/161] enable gpu devices every startup

---
 init/start.sh             | 1 +
 setup-gpu-node-devices.sh | 3 +++
 setup-gpu-node.sh         | 3 ---
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/init/start.sh b/init/start.sh
index 5cacef308..c9669722b 100755
--- a/init/start.sh
+++ b/init/start.sh
@@ -105,6 +105,7 @@ fi
 
 if [[ "${ENV}" == "gpu" ]]; then
   CE_PROP_ENV=gpu
+  /infra/setup-gpu-node-devices.sh
 fi
 
 # shellcheck disable=SC2086
diff --git a/setup-gpu-node-devices.sh b/setup-gpu-node-devices.sh
index a5fecbe3d..d698b953f 100755
--- a/setup-gpu-node-devices.sh
+++ b/setup-gpu-node-devices.sh
@@ -24,3 +24,6 @@ if /sbin/modprobe nvidia-uvm; then
 else
   exit 1
 fi
+
+# /dev/nvidia-modeset
+/bin/nvidia-modprobe -m
diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index 1dff54e4e..75f384b4e 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -110,6 +110,3 @@ update-initramfs -u
 
 # script from https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-verifications
 ./setup-gpu-node-devices.sh
-
-# /dev/nvidia-modeset
-/bin/nvidia-modprobe -m

From b9780bba0220d7babcbc050184b547aab520c5ce Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 5 Nov 2022 22:40:38 +0100
Subject: [PATCH 069/161] surprise!

---
 terraform/ec2.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index a82326b36..5cc1bb2eb 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,7 +2,7 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
-  gpu_image_id     = "ami-06b394f7def740c02"
+  gpu_image_id     = "ami-01d05f46623827076"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 

From e39c63228fe32203bd05f5c5cb75a51621869404 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Poulhi=C3=A8s?= <dkm@kataplop.net>
Date: Sun, 6 Nov 2022 23:06:56 +0100
Subject: [PATCH 070/161] Add c6x config (#864)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add new TI C6x config for C and C++ (c6x-elf).

refs https://github.com/compiler-explorer/compiler-explorer/issues/1806

Signed-off-by: Marc Poulhiès <dkm@kataplop.net>
---
 bin/yaml/cpp.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 0d1f84ee8..de0bdd0a6 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -434,6 +434,12 @@ compilers:
             - 4.9.4
             - 9.5.0
             - 12.2.0
+        c6x:
+          arch_prefix: "tic6x-elf"
+          check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"
+          subdir: c6x
+          targets:
+            - 12.2.0
         loongarch64:
           arch_prefix: "{subdir}-unknown-linux-gnu"
           check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"

From 6a6ec55e04d8187d3b10a55cbfe543c7be6047b6 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 11 Nov 2022 11:11:47 +0100
Subject: [PATCH 071/161] trap later

---
 admin-daily-builds.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/admin-daily-builds.sh b/admin-daily-builds.sh
index 4e35e9d13..30c02341f 100755
--- a/admin-daily-builds.sh
+++ b/admin-daily-builds.sh
@@ -5,7 +5,6 @@ set -exuo pipefail
 finish() {
     ce builder stop
 }
-trap finish EXIT
 
 ce builder status
 ce builder start
@@ -45,6 +44,9 @@ build_rust_libraries() {
 }
 
 build_cpp_libraries
+
+trap finish EXIT
+
 build_rust_libraries
 
 exit ${BUILD_FAILED}

From 3cce9ae8e5c89728a19b6c875f865a540182d170 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Fri, 11 Nov 2022 11:08:09 +0100
Subject: [PATCH 072/161] move openssl to libraries yaml (#866)

---
 bin/yaml/libraries.yaml               | 39 ++++++++++++++++-----------
 update_compilers/install_libraries.sh | 31 ---------------------
 2 files changed, 24 insertions(+), 46 deletions(-)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index 74506f0c6..e52a45613 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -24,9 +24,9 @@ libraries:
       - 1.4.1
       - 1.5.0
       - 1.6.1
-      - name: 1.6.2
-        after_stage_script:
+      - after_stage_script:
         - cp {{resource_dir}}/googlebenchmark_export.h include/benchmark/export.h
+        name: 1.6.2
       type: github
     benri:
       check_file: readme.md
@@ -183,7 +183,7 @@ libraries:
       check_file: CMakeLists.txt
       compression: xz
       depends:
-      - libraries/c++/openssl OpenSSL_1_1_1g
+      - libraries/c++/openssl 1_1_1g
       dir: libs/curl/{{name}}
       extra_cmake_arg:
       - -DOPENSSL_ROOT_DIR=/opt/compiler-explorer/libs/openssl/openssl_1_1_1g/x86_64/opt
@@ -1255,19 +1255,19 @@ libraries:
         targets:
         - trunk
         type: github
-      seqan3:
+      seastar:
         build_type: none
-        check_file: include/seqan3/version.hpp
+        check_file: README.md
         method: nightlyclone
-        repo: seqan/seqan3
+        repo: scylladb/seastar
         targets:
         - trunk
         type: github
-      seastar:
+      seqan3:
         build_type: none
-        check_file: README.md
+        check_file: include/seqan3/version.hpp
         method: nightlyclone
-        repo: scylladb/seastar
+        repo: seqan/seqan3
         targets:
         - trunk
         type: github
@@ -1435,14 +1435,23 @@ libraries:
       - v2.2
       type: s3tarballs
     openssl:
-      build_type: manual
-      check_file: config
+      after_stage_script:
+      - cd {untar_dir}
+      - ./config --prefix="$PWD/x86_64/opt" --openssldir="$PWD/x86_64/ssl"
+      - make
+      - make install
+      - rm "{untar_dir}/x86_64/opt/lib/*.a"
+      build_type: none
+      check_file: x86_64/opt/lib/libssl.so.1.1
+      compression: gz
+      dir: libs/openssl/openssl_{name}
       lib_type: shared
-      repo: openssl/openssl
       targets:
-      - OpenSSL_1_1_1c
-      - OpenSSL_1_1_1g
-      type: github
+      - 1_1_1c
+      - 1_1_1g
+      type: tarballs
+      untar_dir: openssl-OpenSSL_{name}
+      url: https://github.com/openssl/openssl/archive/OpenSSL_{name}.tar.gz
     pegtl:
       check_file: README.md
       repo: taocpp/PEGTL
diff --git a/update_compilers/install_libraries.sh b/update_compilers/install_libraries.sh
index 70b9dc61c..594b6c60a 100755
--- a/update_compilers/install_libraries.sh
+++ b/update_compilers/install_libraries.sh
@@ -20,37 +20,6 @@ fi
 ce_install 'libraries'
 ce_squash 'libraries'
 
-#########################
-# OpenSSL
-
-install_openssl() {
-    for VERSION in "$@"; do
-        local DEST=${OPT}/libs/openssl/openssl_${VERSION}/x86_64/opt
-        if [[ ! -d ${DEST} ]]; then
-            rm -rf /tmp/openssl
-            mkdir -p /tmp/openssl
-            pushd /tmp/openssl
-            fetch "https://github.com/openssl/openssl/archive/OpenSSL_${VERSION}.tar.gz" | tar zxf - --strip-components 1
-
-            setarch i386 ./config -m32 --prefix="${OPT}/libs/openssl/openssl_${VERSION}/x86/opt" --openssldir="${OPT}/libs/openssl/openssl_${VERSION}/x86/ssl"
-            make
-            make install
-            rm "${OPT}/libs/openssl/openssl_${VERSION}/x86/opt/lib/*.a"
-
-            make clean
-            ./config --prefix="${OPT}/libs/openssl/openssl_${VERSION}/x86_64/opt" --openssldir="${OPT}/libs/openssl/openssl_${VERSION}/x86_64/ssl"
-            make
-            make install
-            rm "${OPT}/libs/openssl/openssl_${VERSION}/x86_64/opt/lib/*.a"
-            popd
-
-            rm -rf /tmp/openssl
-        fi
-    done
-}
-
-install_openssl 1_1_1c 1_1_1g
-
 #########################
 # cs50
 

From d70d51a5a97a5f8e9549886f3ada039a9d73e788 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= <ruben@rinconblanco.es>
Date: Fri, 11 Nov 2022 11:08:26 +0100
Subject: [PATCH 073/161] Add fmt 9.0.0 & 9.1.0 (#867)

---
 bin/yaml/libraries.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index e52a45613..c6dd2561c 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -357,6 +357,8 @@ libraries:
       - 7.1.3
       - 8.0.1
       - 8.1.1
+      - 9.0.0
+      - 9.1.0
       type: github
     gemmlowp:
       build_type: none

From 90293eb5a868f4e681adb7e74c76703314474714 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 11 Nov 2022 11:16:26 +0100
Subject: [PATCH 074/161] nevermind, not the issue

---
 admin-daily-builds.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/admin-daily-builds.sh b/admin-daily-builds.sh
index 30c02341f..4e35e9d13 100755
--- a/admin-daily-builds.sh
+++ b/admin-daily-builds.sh
@@ -5,6 +5,7 @@ set -exuo pipefail
 finish() {
     ce builder stop
 }
+trap finish EXIT
 
 ce builder status
 ce builder start
@@ -44,9 +45,6 @@ build_rust_libraries() {
 }
 
 build_cpp_libraries
-
-trap finish EXIT
-
 build_rust_libraries
 
 exit ${BUILD_FAILED}

From 0a6f23d8d7fccc93291e9d44b1853521480c325f Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 11 Nov 2022 11:23:50 +0100
Subject: [PATCH 075/161] hint type, but python typesystem is useless

---
 bin/lib/rust_library_builder.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/bin/lib/rust_library_builder.py b/bin/lib/rust_library_builder.py
index ad681e8f8..488f877f0 100644
--- a/bin/lib/rust_library_builder.py
+++ b/bin/lib/rust_library_builder.py
@@ -15,6 +15,7 @@
 
 # from packaging import version
 import requests
+from lib.installation_context import InstallationContext
 
 from lib.rust_crates import RustCrate
 
@@ -62,7 +63,13 @@ def open_script(script: Path) -> Generator[TextIO, None, None]:
 
 class RustLibraryBuilder:
     def __init__(
-        self, logger, language: str, libname: str, target_name: str, install_context, buildconfig: LibraryBuildConfig
+        self,
+        logger,
+        language: str,
+        libname: str,
+        target_name: str,
+        install_context: InstallationContext,
+        buildconfig: LibraryBuildConfig,
     ):
         self.logger = logger
         self.language = language

From 24e0c100cadeb870ba017b7327ab43dbc38e7846 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 11 Nov 2022 11:32:42 +0100
Subject: [PATCH 076/161] fixes

---
 bin/lib/rust_library_builder.py | 63 ++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 29 deletions(-)

diff --git a/bin/lib/rust_library_builder.py b/bin/lib/rust_library_builder.py
index 488f877f0..8e814d036 100644
--- a/bin/lib/rust_library_builder.py
+++ b/bin/lib/rust_library_builder.py
@@ -15,6 +15,7 @@
 
 # from packaging import version
 import requests
+from lib.staging import StagingDir
 from lib.installation_context import InstallationContext
 
 from lib.rust_crates import RustCrate
@@ -374,28 +375,28 @@ def set_as_uploaded(self, buildfolder, source_folder, build_method):
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
 
-    def clone_branch(self, dest):
+    def clone_branch(self, dest, staging: StagingDir):
         subprocess.check_call(
             ["git", "clone", "-q", f"{self.buildconfig.domainurl}/{self.buildconfig.repo}.git", dest],
-            cwd=self.install_context.staging,
+            cwd=staging.path,
         )
-        subprocess.check_call(["git", "-C", dest, "checkout", "-q", self.target_name], cwd=self.install_context.staging)
+        subprocess.check_call(["git", "-C", dest, "checkout", "-q", self.target_name], cwd=staging.path)
 
-    def download_library(self, build_folder, source_folder):
+    def download_library(self, build_folder, source_folder, staging: StagingDir):
         if not os.path.exists(os.path.join(source_folder, "Cargo.toml")):
             self.logger.info(f"Downloading sources for {self.libname}/{self.target_name}")
 
             if self.buildconfig.repo:
-                self.clone_branch(source_folder)
+                self.clone_branch(source_folder, staging)
             else:
                 crate = RustCrate(self.libname, self.target_name)
                 url = crate.GetDownloadUrl()
                 tar_cmd = ["tar", "zxf", "-"]
                 tar_cmd += ["--strip-components", "1"]
-                self.install_context.fetch_url_and_pipe_to(f"{url}", tar_cmd, source_folder)
+                self.install_context.fetch_url_and_pipe_to(staging, f"{url}", tar_cmd, source_folder)
 
-    def get_source_folder(self):
-        source_folder = os.path.join(self.install_context.staging, f"source_{self.libname}_{self.target_name}")
+    def get_source_folder(self, staging: StagingDir):
+        source_folder = os.path.join(staging.path, f"source_{self.libname}_{self.target_name}")
         if not source_folder in self.cached_source_folders:
             if not os.path.exists(source_folder):
                 os.mkdir(source_folder)
@@ -417,24 +418,8 @@ def makebuildfor(
         flagscombination,
         ld_path,
     ):
-        build_method = dict({"build_method": "--all-features", "linker": "/opt/compiler-explorer/gcc-11.1.0"})
-        build_status = self.makebuildfor_by_method(
-            compiler,
-            options,
-            exe,
-            compiler_type,
-            toolchain,
-            buildos,
-            buildtype,
-            arch,
-            stdver,
-            stdlib,
-            flagscombination,
-            ld_path,
-            build_method,
-        )
-        if build_status == BuildStatus.Failed:
-            build_method = dict({"build_method": "", "linker": "/opt/compiler-explorer/gcc-11.1.0"})
+        with self.install_context.new_staging_dir() as staging:
+            build_method = dict({"build_method": "--all-features", "linker": "/opt/compiler-explorer/gcc-11.1.0"})
             build_status = self.makebuildfor_by_method(
                 compiler,
                 options,
@@ -449,7 +434,26 @@ def makebuildfor(
                 flagscombination,
                 ld_path,
                 build_method,
+                staging,
             )
+            if build_status == BuildStatus.Failed:
+                build_method = dict({"build_method": "", "linker": "/opt/compiler-explorer/gcc-11.1.0"})
+                build_status = self.makebuildfor_by_method(
+                    compiler,
+                    options,
+                    exe,
+                    compiler_type,
+                    toolchain,
+                    buildos,
+                    buildtype,
+                    arch,
+                    stdver,
+                    stdlib,
+                    flagscombination,
+                    ld_path,
+                    build_method,
+                    staging,
+                )
 
         return build_status
 
@@ -468,12 +472,13 @@ def makebuildfor_by_method(
         flagscombination,
         ld_path,
         build_method,
+        staging: StagingDir,
     ):
         combined_hash = self.makebuildhash(
             compiler, options, toolchain, buildos, buildtype, arch, stdver, stdlib, flagscombination
         )
 
-        build_folder = os.path.join(self.install_context.staging, combined_hash)
+        build_folder = os.path.join(staging.path, combined_hash)
         if os.path.exists(build_folder):
             shutil.rmtree(build_folder, ignore_errors=True)
         os.makedirs(build_folder, exist_ok=True)
@@ -482,7 +487,7 @@ def makebuildfor_by_method(
 
         real_build_folder = os.path.join(build_folder, "build")
 
-        source_folder = self.get_source_folder()
+        source_folder = self.get_source_folder(staging)
 
         self.writeconanfile(build_folder)
 
@@ -513,7 +518,7 @@ def makebuildfor_by_method(
             if not self.forcebuild:
                 return BuildStatus.Skipped
 
-        self.download_library(build_folder, source_folder)
+        self.download_library(build_folder, source_folder, staging)
 
         if not self.install_context.dry_run and not self.conanserverproxy_token:
             self.conanproxy_login()

From ce85ab2a189af07698c64323631ea38ee2182f40 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 11 Nov 2022 12:23:02 +0100
Subject: [PATCH 077/161] resilient post

---
 bin/lib/rust_library_builder.py | 45 +++++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 11 deletions(-)

diff --git a/bin/lib/rust_library_builder.py b/bin/lib/rust_library_builder.py
index 8e814d036..61c9ecbd0 100644
--- a/bin/lib/rust_library_builder.py
+++ b/bin/lib/rust_library_builder.py
@@ -12,6 +12,7 @@
 from enum import Enum, unique
 from pathlib import Path
 from typing import Dict, Any, List, Optional, Generator, TextIO
+from urllib3.exceptions import ProtocolError
 
 # from packaging import version
 import requests
@@ -262,15 +263,38 @@ def get_conan_hash(self, buildfolder: str) -> Optional[str]:
                 return match[1]
         return None
 
+    def resil_post(self, url, json_data, headers=None):
+        request = None
+        retries = 3
+        last_error = ""
+        while retries > 0:
+            try:
+                if headers != None:
+                    request = requests.post(url, data=json_data, headers=headers, timeout=_TIMEOUT)
+                else:
+                    request = requests.post(
+                        url, data=json_data, headers={"Content-Type": "application/json"}, timeout=_TIMEOUT
+                    )
+
+                retries = 0
+            except ProtocolError as e:
+                last_error = e
+                retries = retries - 1
+
+        if request == None:
+            request = {"ok": False, "text": last_error}
+
+        return request
+
     def conanproxy_login(self):
         url = f"{conanserver_url}/login"
 
         login_body = defaultdict(lambda: [])
         login_body["password"] = get_ssm_param("/compiler-explorer/conanpwd")
 
-        request = requests.post(
-            url, data=json.dumps(login_body), headers={"Content-Type": "application/json"}, timeout=_TIMEOUT
-        )
+        req_data = json.dumps(login_body)
+
+        request = self.resil_post(url, req_data)
         if not request.ok:
             self.logger.info(request.text)
             raise RuntimeError(f"Post failure for {url}: {request}")
@@ -303,7 +327,8 @@ def save_build_logging(self, builtok, build_folder, source_folder):
 
         headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token}
 
-        request = requests.post(url, data=json.dumps(buildparameters_copy), headers=headers, timeout=_TIMEOUT)
+        req_data = json.dumps(buildparameters_copy)
+        request = self.resil_post(url, req_data, headers)
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
 
@@ -328,12 +353,10 @@ def get_commit_hash(self) -> str:
         return self.target_name
 
     def has_failed_before(self):
-        headers = {"Content-Type": "application/json"}
-
         url = f"{conanserver_url}/hasfailedbefore"
-        request = requests.post(
-            url, data=json.dumps(self.current_buildparameters_obj), headers=headers, timeout=_TIMEOUT
-        )
+        req_data = json.dumps(self.current_buildparameters_obj)
+
+        request = self.resil_post(url, req_data)
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
         else:
@@ -369,9 +392,9 @@ def set_as_uploaded(self, buildfolder, source_folder, build_method):
         self.logger.info(annotations)
 
         headers = {"Content-Type": "application/json", "Authorization": "Bearer " + self.conanserverproxy_token}
-
         url = f"{conanserver_url}/annotations/{self.libname}/{self.target_name}/{conanhash}"
-        request = requests.post(url, data=json.dumps(annotations), headers=headers, timeout=_TIMEOUT)
+
+        request = self.resil_post(url, json.dumps(annotations), headers)
         if not request.ok:
             raise RuntimeError(f"Post failure for {url}: {request}")
 

From da3189e90c3a1167043be0176fcf76e9dd23525f Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 12 Nov 2022 14:40:39 +0100
Subject: [PATCH 078/161] dont ignore

---
 admin-daily-builds.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/admin-daily-builds.sh b/admin-daily-builds.sh
index 4e35e9d13..ae30806d6 100755
--- a/admin-daily-builds.sh
+++ b/admin-daily-builds.sh
@@ -22,7 +22,6 @@ build_cpp_libraries() {
     ce builder exec -- sudo docker run --rm --name "${BUILD_NAME}.build" \
         -v/home/ubuntu/.s3cfg:/root/.s3cfg:ro \
         -v/opt:/opt:ro \
-        -e 'LOGSPOUT=ignore' \
         -e "CONAN_PASSWORD=${CONAN_PASSWORD}" \
         "compilerexplorer/library-builder" \
         bash "${COMMAND}" "c++" "all" "all"
@@ -38,7 +37,6 @@ build_rust_libraries() {
     ce builder exec -- sudo docker run --rm --name "${BUILD_NAME}.build" \
         -v/home/ubuntu/.s3cfg:/root/.s3cfg:ro \
         -v/opt:/opt:ro \
-        -e 'LOGSPOUT=ignore' \
         -e "CONAN_PASSWORD=${CONAN_PASSWORD}" \
         "compilerexplorer/library-builder" \
         bash "${COMMAND}" "rust" "all" "all"

From 8b3412e029f8b697ec1a22237e436a6b73970c16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Poulhi=C3=A8s?= <dkm@kataplop.net>
Date: Fri, 11 Nov 2022 21:01:16 +0100
Subject: [PATCH 079/161] Add sparc, sparc64 and sparc-leon compilers (#868)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

3 new gcc cross compilers for sparc, sparc64 and sparc-leon.

refs https://github.com/compiler-explorer/compiler-explorer/issues/266

Signed-off-by: Marc Poulhiès <dkm@kataplop.net>
---
 bin/yaml/cpp.yaml | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index de0bdd0a6..e3bd403d0 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -440,6 +440,24 @@ compilers:
           subdir: c6x
           targets:
             - 12.2.0
+        sparc:
+          arch_prefix: "sparc-unknown-linux-gnu"
+          check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"
+          subdir: sparc
+          targets:
+            - 12.2.0
+        sparc64:
+          arch_prefix: "sparc64-multilib-linux-gnu"
+          check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"
+          subdir: sparc64
+          targets:
+            - 12.2.0
+        sparc-leon:
+          arch_prefix: "sparc-leon-linux-uclibc"
+          check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"
+          subdir: sparc-leon
+          targets:
+            - 12.2.0
         loongarch64:
           arch_prefix: "{subdir}-unknown-linux-gnu"
           check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"

From 06e2f900a9fd8ddc252e559ff8a83941cdae06dc Mon Sep 17 00:00:00 2001
From: Cory Bloor <Cordell.Bloor@amd.com>
Date: Fri, 11 Nov 2022 20:06:32 -0700
Subject: [PATCH 080/161] Add ROCm 5.3.2 (#857)

---
 bin/yaml/cpp.yaml       | 1 +
 bin/yaml/libraries.yaml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index e3bd403d0..42903032b 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -637,6 +637,7 @@ compilers:
           - 5.0.2
           - 5.1.3
           - 5.2.3
+          - 5.3.2
     ellccs:
       check_exe: bin/clang++ --version
       dir: ellcc-{name}
diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index c6dd2561c..d7017dda0 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -469,6 +469,7 @@ libraries:
       - 5.0.2
       - 5.1.3
       - 5.2.3
+      - 5.3.2
       type: s3tarballs
       untar_dir: hip-amd-rocm-{{name}}
     jsoncons:

From d8029e5344ece8130d73b6738b2cba841f7c57be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Poulhi=C3=A8s?= <dkm@kataplop.net>
Date: Sat, 12 Nov 2022 12:21:13 +0100
Subject: [PATCH 081/161] Add new version of Kalray's KVX compiler (#870)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add latest 4.10.0 compiler.

refs https://github.com/compiler-explorer/compiler-explorer/issues/4280

Signed-off-by: Marc Poulhiès <dkm@kataplop.net>
---
 bin/yaml/cpp.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 42903032b..60368899f 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -418,6 +418,7 @@ compilers:
             - 4.6.0
             - 4.8.0
             - 4.9.0
+            - 4.10.0
         s390x:
           arch_prefix: "{subdir}-ibm-linux-gnu"
           check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"

From 8da75c897815b91bebce3ce03bdb44920455bdda Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 12 Nov 2022 14:46:26 +0100
Subject: [PATCH 082/161] attempt to add logspout to librarybuilder

---
 admin-daily-builds.sh | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/admin-daily-builds.sh b/admin-daily-builds.sh
index ae30806d6..16acd20a1 100755
--- a/admin-daily-builds.sh
+++ b/admin-daily-builds.sh
@@ -12,6 +12,24 @@ ce builder start
 
 BUILD_FAILED=0
 
+init_logspout() {
+    local LOG_DEST_HOST
+    LOG_DEST_HOST=$(aws ssm get-parameter --name /compiler-explorer/logHost | jq -r .Parameter.Value)
+    local LOG_DEST_PORT
+    LOG_DEST_PORT=$(aws ssm get-parameter --name /compiler-explorer/logPort | jq -r .Parameter.Value)
+
+    ce builder exec -- sudo docker pull gliderlabs/logspout:latest
+
+    ce builder exec -- sudo docker stop logspout || true
+    ce builder exec -- sudo docker rm logspout || true
+
+    ce builder exec -- sudo docker run --name logspout \
+        -d \
+        -v=/var/run/docker.sock:/tmp/docker.sock \
+        -e SYSLOG_HOSTNAME=librarybuilder \
+        gliderlabs/logspout "syslog+tls://${LOG_DEST_HOST}:${LOG_DEST_PORT}"
+}
+
 build_cpp_libraries() {
     local BUILD_NAME=librarycpp
     local COMMAND=build.sh
@@ -42,6 +60,7 @@ build_rust_libraries() {
         bash "${COMMAND}" "rust" "all" "all"
 }
 
+init_logspout
 build_cpp_libraries
 build_rust_libraries
 

From d9aa5d605e5d7793670173b31c9224de888112dc Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 12 Nov 2022 14:50:40 +0100
Subject: [PATCH 083/161] fix paramnames

---
 admin-daily-builds.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/admin-daily-builds.sh b/admin-daily-builds.sh
index 16acd20a1..2460995d8 100755
--- a/admin-daily-builds.sh
+++ b/admin-daily-builds.sh
@@ -14,9 +14,9 @@ BUILD_FAILED=0
 
 init_logspout() {
     local LOG_DEST_HOST
-    LOG_DEST_HOST=$(aws ssm get-parameter --name /compiler-explorer/logHost | jq -r .Parameter.Value)
+    LOG_DEST_HOST=$(aws ssm get-parameter --name /compiler-explorer/logDestHost | jq -r .Parameter.Value)
     local LOG_DEST_PORT
-    LOG_DEST_PORT=$(aws ssm get-parameter --name /compiler-explorer/logPort | jq -r .Parameter.Value)
+    LOG_DEST_PORT=$(aws ssm get-parameter --name /compiler-explorer/logDestPort | jq -r .Parameter.Value)
 
     ce builder exec -- sudo docker pull gliderlabs/logspout:latest
 

From e0cf86efd29e89d309adb725a7651e30cbf8edc0 Mon Sep 17 00:00:00 2001
From: Christian Clauss <cclauss@me.com>
Date: Mon, 14 Nov 2022 02:30:20 +0100
Subject: [PATCH 084/161] Add Python 3.11 to the testing (#865)

---
 .github/workflows/python-tests.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
index 0165f1548..e248df441 100644
--- a/.github/workflows/python-tests.yaml
+++ b/.github/workflows/python-tests.yaml
@@ -10,6 +10,7 @@ jobs:
         python-version:
           - 3.9
           - '3.10'
+          - 3.11
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}

From c72c42b10994e2918fbfa53139dea9a5b6dae4ee Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 19:52:50 -0600
Subject: [PATCH 085/161] Alternative packer setup for gpu

---
 setup-gpu-node-devices.sh |  2 +
 setup-gpu-node.sh         | 94 +--------------------------------------
 2 files changed, 3 insertions(+), 93 deletions(-)

diff --git a/setup-gpu-node-devices.sh b/setup-gpu-node-devices.sh
index d698b953f..d6982f5fe 100755
--- a/setup-gpu-node-devices.sh
+++ b/setup-gpu-node-devices.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -euo pipefail
+
 if /sbin/modprobe nvidia; then
   # Count the number of NVIDIA controllers found.
   NVDEVS=$(lspci | grep -i NVIDIA)
diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index 75f384b4e..6eb0756a1 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -4,99 +4,7 @@ set -exuo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-env EXTRA_NFS_ARGS=",ro" "${DIR}/setup-common.sh"
-
-apt-get -y update
-apt-get -y install software-properties-common
-dpkg --add-architecture i386
-curl -s https://dl.winehq.org/wine-builds/winehq.key | apt-key add -
-# TODO at some point see if we can upgrade wine
-apt-add-repository 'deb https://dl.winehq.org/wine-builds/ubuntu/ bionic main'
-apt-get install -y \
-    binutils-multiarch \
-    bison \
-    bzip2 \
-    cgroup-tools \
-    curl \
-    dos2unix \
-    file \
-    flex \
-    g++ \
-    gawk \
-    gcc \
-    gnat \
-    jq \
-    libapparmor-dev \
-    libc6-dev-i386 \
-    libdatetime-perl \
-    libelf-dev \
-    libgmp3-dev \
-    libprotobuf-dev \
-    libnl-route-3-dev \
-    libwww-perl \
-    linux-libc-dev \
-    make \
-    nfs-common \
-    nginx \
-    patch \
-    pkg-config \
-    protobuf-compiler \
-    python3-pip \
-    python3.8-venv \
-    s3cmd \
-    subversion \
-    texinfo \
-    unzip \
-    wget \
-    winehq-stable=4.0.3~bionic \
-    wine-stable=4.0.3~bionic \
-    wine-stable-amd64=4.0.3~bionic \
-    wine-stable-i386=4.0.3~bionic \
-    xz-utils
-
-pushd /tmp
-git clone https://github.com/apmorton/firejail.git firejail-apmorton
-cd firejail-apmorton
-git checkout 0.9.58.2-ce-patch.1
-./configure --enable-apparmor --prefix /usr/local/firejail-0.9.58.2-ce-patch.1
-make "-j$(nproc)"
-make install
-popd
-
-ln -s /usr/local/firejail-0.9.58.2-ce-patch.1/bin/firejail /usr/local/bin
-
-pushd /tmp
-git clone https://github.com/netblue30/firejail.git
-cd firejail
-git checkout 0.9.70
-./configure --enable-apparmor --prefix /usr/local/firejail-0.9.70
-make "-j$(nproc)"
-make install
-popd
-
-pushd /tmp
-git clone --recursive --branch 3.0 https://github.com/google/nsjail.git
-cd nsjail
-make "-j$(nproc)"
-cp nsjail /usr/local/bin/nsjail
-popd
-
-
-pushd /opt
-# node.js
-TARGET_NODE_VERSION=v16.17.1
-echo "Installing node ${TARGET_NODE_VERSION}"
-curl -sL "https://nodejs.org/dist/${TARGET_NODE_VERSION}/node-${TARGET_NODE_VERSION}-linux-x64.tar.xz" | tar xJf - && mv node-${TARGET_NODE_VERSION}-linux-x64 node
-popd
-
-cp nginx/nginx-gpu.conf /etc/nginx/nginx.conf
-systemctl restart nginx
-
-cp /infra/init/compiler-explorer.service /lib/systemd/system/compiler-explorer.service
-systemctl daemon-reload
-systemctl enable compiler-explorer
-
-adduser --system --group ce
+"${DIR}/setup-node.sh"
 
 # setup nvidia drivers https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-nouveau-ubuntu
 

From 903a85529e9218d21a32c948710ce05d30d271f6 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 19:55:18 -0600
Subject: [PATCH 086/161] Alternative packer setup for gpu

---
 setup-gpu-node.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index 6eb0756a1..febed37e0 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -4,7 +4,7 @@ set -exuo pipefail
 
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
-"${DIR}/setup-node.sh"
+bash "${DIR}/setup-node.sh"
 
 # setup nvidia drivers https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-nouveau-ubuntu
 
@@ -17,4 +17,4 @@ echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/black
 update-initramfs -u
 
 # script from https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-verifications
-./setup-gpu-node-devices.sh
+"${DIR}/setup-gpu-node-devices.sh"

From 96b672dd14dea9c4384e404bfc92fa7c3e5a6ca6 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 20:07:20 -0600
Subject: [PATCH 087/161] Alternative packer setup for gpu

---
 setup-gpu-node-devices.sh | 2 --
 setup-gpu-node.sh         | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/setup-gpu-node-devices.sh b/setup-gpu-node-devices.sh
index d6982f5fe..d698b953f 100755
--- a/setup-gpu-node-devices.sh
+++ b/setup-gpu-node-devices.sh
@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -euo pipefail
-
 if /sbin/modprobe nvidia; then
   # Count the number of NVIDIA controllers found.
   NVDEVS=$(lspci | grep -i NVIDIA)
diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index febed37e0..cf4579730 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -11,6 +11,7 @@ bash "${DIR}/setup-node.sh"
 pushd /tmp
 wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
 sh cuda_11.8.0_520.61.05_linux.run --silent --driver
+rm cuda_11.8.0_520.61.05_linux.run
 popd
 
 echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf

From 6aaa452019264902f81b0ffca86b88b17b9fd77c Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 20:10:56 -0600
Subject: [PATCH 088/161] more tweaks

---
 init/start.sh        |  5 ----
 nginx/nginx-gpu.conf | 67 --------------------------------------------
 site-gpu.sh          |  2 ++
 3 files changed, 2 insertions(+), 72 deletions(-)
 delete mode 100644 nginx/nginx-gpu.conf

diff --git a/init/start.sh b/init/start.sh
index c9669722b..fde1058a8 100755
--- a/init/start.sh
+++ b/init/start.sh
@@ -103,11 +103,6 @@ if [[ "${ENV}" == "runner" ]]; then
   exit
 fi
 
-if [[ "${ENV}" == "gpu" ]]; then
-  CE_PROP_ENV=gpu
-  /infra/setup-gpu-node-devices.sh
-fi
-
 # shellcheck disable=SC2086
 exec sudo -u ${CE_USER} -H --preserve-env=NODE_ENV -- \
     /opt/node/bin/node \
diff --git a/nginx/nginx-gpu.conf b/nginx/nginx-gpu.conf
deleted file mode 100644
index d7faa2765..000000000
--- a/nginx/nginx-gpu.conf
+++ /dev/null
@@ -1,67 +0,0 @@
-user www-data;
-worker_processes 1;
-pid /run/nginx.pid;
-
-events {
-	worker_connections 768;
-	# multi_accept on;
-}
-
-http {
-	sendfile on;
-	tcp_nopush on;
-	tcp_nodelay on;
-	keepalive_timeout 65;
-	types_hash_max_size 2048;
-
-	include /etc/nginx/mime.types;
-	default_type application/octet-stream;
-
-	##
-	# SSL Settings
-	##
-
-	ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
-	ssl_prefer_server_ciphers on;
-
-	##
-	# Logging Settings
-	##
-
-	access_log /var/log/nginx/access.log;
-	error_log /var/log/nginx/error.log;
-
-	##
-	# Gzip Settings
-	##
-
-	gzip on;
-	gzip_disable "msie6";
-    gzip_proxied any;
-    gzip_http_version 1.1;
-    gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
-    set_real_ip_from 172.30.0.0/16;
-    real_ip_header X-Forwarded-For;
-    client_max_body_size 10m;
-
-    server {
-        index index.html;
-        access_log /var/log/nginx/all.log;
-        error_log /var/log/nginx/all.err;
-		listen 1081;
-
-        location / {
-            proxy_pass http://localhost:10240;
-            proxy_set_header Host $host;
-
-            # uncomment the following line if you run your own instance
-            # proxy_set_header X-Forwarded-Proto $scheme;
-
-            # See https://ma.ttias.be/nginx-proxy-upstream-sent-big-header-reading-response-header-upstream/
-            # and https://github.com/mattgodbolt/compiler-explorer/issues/1423
-            proxy_buffer_size          128k;
-            proxy_buffers              4 256k;
-            proxy_busy_buffers_size    256k;
-        }
-    }
-}
diff --git a/site-gpu.sh b/site-gpu.sh
index 0e4014693..2c4b60be0 100644
--- a/site-gpu.sh
+++ b/site-gpu.sh
@@ -2,3 +2,5 @@
 # Sourced when running in gpu mode
 export BRANCH=gpu
 export NODE_ENV=production
+export CE_PROP_ENV=gpu
+/infra/setup-gpu-node-devices.sh

From adec3b50c7ca840489b7d97899417a219d71835d Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 20:46:47 -0600
Subject: [PATCH 089/161] add gpu env

---
 bin/lib/env.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/lib/env.py b/bin/lib/env.py
index a12b221dc..4dce7bc66 100644
--- a/bin/lib/env.py
+++ b/bin/lib/env.py
@@ -6,6 +6,7 @@ class Environment(Enum):
     PROD = "prod"
     BETA = "beta"
     STAGING = "staging"
+    GPU = "gpu"
     RUNNER = "runner"
 
 

From 8435826e17989e3d5c1473a9df068bf65193604a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 21:02:28 -0600
Subject: [PATCH 090/161] WIP on gpu refactor

---
 init/start.sh               |  4 ++--
 setup-gpu-node.sh           |  6 +++---
 site-gpu.sh                 |  2 +-
 terraform/alb.tf            | 35 +++++++++++++++++++++++++++++++++-
 terraform/asg.tf            | 23 ++++++++++++++++++++++
 terraform/cloudfront_gpu.tf |  1 +
 terraform/dynamodb.tf       | 27 --------------------------
 terraform/ec2.tf            |  5 +++--
 terraform/lc.tf             | 38 ++++++++++++++++++++++++++++++++++---
 terraform/security.tf       |  3 ++-
 terraform/tg.tf             |  2 ++
 11 files changed, 106 insertions(+), 40 deletions(-)

diff --git a/init/start.sh b/init/start.sh
index fde1058a8..4362014d6 100755
--- a/init/start.sh
+++ b/init/start.sh
@@ -8,7 +8,7 @@ CE_USER=ce
 DEPLOY_DIR=${PWD}/.deploy
 COMPILERS_ARG=
 COMPILERS_FILE=$DEPLOY_DIR/discovered-compilers.json
-CE_PROP_ENV=amazon
+CE_PROP_ENV=--env amazon
 
 echo Running in environment "${ENV}"
 # shellcheck disable=SC1090
@@ -111,7 +111,7 @@ exec sudo -u ${CE_USER} -H --preserve-env=NODE_ENV -- \
     --suppressConsoleLog \
     --logHost "${LOG_DEST_HOST}" \
     --logPort "${LOG_DEST_PORT}" \
-    --env ${CE_PROP_ENV} \
+    ${CE_PROP_ENV} \
     --port 10240 \
     --metricsPort 10241 \
     --loki "http://127.0.0.1:3500" \
diff --git a/setup-gpu-node.sh b/setup-gpu-node.sh
index cf4579730..81177a895 100644
--- a/setup-gpu-node.sh
+++ b/setup-gpu-node.sh
@@ -9,9 +9,9 @@ bash "${DIR}/setup-node.sh"
 # setup nvidia drivers https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#runfile-nouveau-ubuntu
 
 pushd /tmp
-wget https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
-sh cuda_11.8.0_520.61.05_linux.run --silent --driver
-rm cuda_11.8.0_520.61.05_linux.run
+curl -sL https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run -o install.run
+sh install.run --silent --driver
+rm install.run
 popd
 
 echo -e "blacklist nouveau\noptions nouveau modeset=0\n" > /etc/modprobe.d/blacklist-nouveau.conf
diff --git a/site-gpu.sh b/site-gpu.sh
index 2c4b60be0..372176c97 100644
--- a/site-gpu.sh
+++ b/site-gpu.sh
@@ -2,5 +2,5 @@
 # Sourced when running in gpu mode
 export BRANCH=gpu
 export NODE_ENV=production
-export CE_PROP_ENV=gpu
+export CE_PROP_ENV=${CE_PROP_ENV} --env gpu
 /infra/setup-gpu-node-devices.sh
diff --git a/terraform/alb.tf b/terraform/alb.tf
index 6340e194d..9b73daa41 100644
--- a/terraform/alb.tf
+++ b/terraform/alb.tf
@@ -27,6 +27,7 @@ resource "aws_alb_listener" "compiler-explorer-alb-listen-http" {
   protocol          = "HTTP"
 }
 
+// TODO clean this repetition up
 resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-http-beta" {
   priority = 1
   action {
@@ -57,6 +58,22 @@ resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-http-staging" {
   listener_arn = aws_alb_listener.compiler-explorer-alb-listen-http.arn
 }
 
+resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-http-gpu" {
+  priority = 3
+  action {
+    type             = "forward"
+    target_group_arn = aws_alb_target_group.ce["gpu"].arn
+  }
+  condition {
+    path_pattern {
+      values = [
+        "/gpu*"
+      ]
+    }
+  }
+  listener_arn = aws_alb_listener.compiler-explorer-alb-listen-http.arn
+}
+
 resource "aws_alb_listener" "compiler-explorer-alb-listen-https" {
   default_action {
     type             = "forward"
@@ -101,6 +118,22 @@ resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-https-staging" {
   listener_arn = aws_alb_listener.compiler-explorer-alb-listen-https.arn
 }
 
+resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-https-gpu" {
+  priority = 3
+  action {
+    type             = "forward"
+    target_group_arn = aws_alb_target_group.ce["gpu"].arn
+  }
+  condition {
+    path_pattern {
+      values = [
+        "/gpu*"
+      ]
+    }
+  }
+  listener_arn = aws_alb_listener.compiler-explorer-alb-listen-https.arn
+}
+
 
 resource "aws_alb_listener" "ceconan-alb-listen-http" {
   default_action {
@@ -137,7 +170,7 @@ resource "aws_alb_target_group_attachment" "lambda-stats-endpoint" {
 }
 
 resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-https-lambda" {
-  priority = 3
+  priority = 4
   action {
     type             = "forward"
     target_group_arn = aws_alb_target_group.lambda.arn
diff --git a/terraform/asg.tf b/terraform/asg.tf
index fee590aad..789ae0cf2 100644
--- a/terraform/asg.tf
+++ b/terraform/asg.tf
@@ -3,6 +3,9 @@ locals {
   // As of Oct 3 2022, startups started taking >2m
   grace_period = 160
   cooldown     = 180
+  // Current c5 on-demand price is 0.085. Yearly pre-pay is 0.05 (so this is same as prepaying a year)
+  // Historically we pay ~ 0.035
+  spot_price        = "0.05"
 }
 
 
@@ -125,3 +128,23 @@ resource "aws_autoscaling_group" "staging" {
 
   target_group_arns = [aws_alb_target_group.ce["staging"].arn]
 }
+
+resource "aws_autoscaling_group" "gpu" {
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  default_cooldown          = local.cooldown
+  health_check_grace_period = local.grace_period
+  health_check_type         = "ELB"
+  launch_template {
+    id      = aws_launch_template.CompilerExplorer-prod-gpu.id
+    version = "$Latest"
+  }
+  max_size            = 2
+  min_size            = 0
+  name                = "gpu"
+  vpc_zone_identifier = local.subnets
+
+  target_group_arns = [aws_alb_target_group.ce["gpu"].arn]
+}
diff --git a/terraform/cloudfront_gpu.tf b/terraform/cloudfront_gpu.tf
index 4ad8712f8..d9f52476f 100644
--- a/terraform/cloudfront_gpu.tf
+++ b/terraform/cloudfront_gpu.tf
@@ -1,3 +1,4 @@
+// TODO delete me
 resource "aws_cloudfront_distribution" "gpu-compiler-explorer-com" {
   origin {
     domain_name = "compiler-explorer.s3.amazonaws.com"
diff --git a/terraform/dynamodb.tf b/terraform/dynamodb.tf
index 5793ab290..49661debd 100644
--- a/terraform/dynamodb.tf
+++ b/terraform/dynamodb.tf
@@ -27,33 +27,6 @@ resource "aws_dynamodb_table" "links" {
   }
 }
 
-resource "aws_dynamodb_table" "gpulinks" {
-  name = "gpulinks"
-  lifecycle {
-    ignore_changes = [
-      read_capacity,
-      write_capacity
-    ]
-    prevent_destroy = true
-  }
-  billing_mode   = "PAY_PER_REQUEST"
-  hash_key       = "prefix"
-  range_key      = "unique_subhash"
-
-  attribute {
-    name = "prefix"
-    type = "S"
-  }
-  attribute {
-    name = "unique_subhash"
-    type = "S"
-  }
-
-  point_in_time_recovery {
-    enabled = true
-  }
-}
-
 resource "aws_dynamodb_table" "versionslog" {
   name = "versionslog"
   lifecycle {
diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index 5cc1bb2eb..1a59d3698 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,7 +2,7 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
-  gpu_image_id     = "ami-01d05f46623827076"
+  gpu_image_id_old     = "ami-01d05f46623827076"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 
@@ -131,8 +131,9 @@ resource "aws_instance" "CERunner" {
   }
 }
 
+// TODO remove this
 resource "aws_instance" "GPUNode" {
-  ami                         = local.gpu_image_id
+  ami                         = local.gpu_image_id_old
   iam_instance_profile        = aws_iam_instance_profile.CompilerExplorerRole.name
   ebs_optimized               = false
   instance_type               = "g4dn.xlarge"
diff --git a/terraform/lc.tf b/terraform/lc.tf
index dc50f7115..4341afd9e 100644
--- a/terraform/lc.tf
+++ b/terraform/lc.tf
@@ -2,11 +2,10 @@ locals {
   image_id          = "ami-08b5bae0de035991b"
   staging_image_id  = "ami-08b5bae0de035991b"
   beta_image_id     = "ami-08b5bae0de035991b"
+  gpu_image_id      = "ami-0ea161f7cd222e436"
   staging_user_data = base64encode("staging")
   beta_user_data    = base64encode("beta")
-  // Current c5 on-demand price is 0.085. Yearly pre-pay is 0.05 (so this is same as prepaying a year)
-  // Historically we pay ~ 0.035
-  spot_price        = "0.05"
+  gpu_user_data     = base64encode("gpu")
 }
 
 resource "aws_launch_template" "CompilerExplorer-beta" {
@@ -75,6 +74,39 @@ resource "aws_launch_template" "CompilerExplorer-staging" {
   }
 }
 
+resource "aws_launch_template" "CompilerExplorer-prod-gpu" {
+  name          = "ce-prod-gpu"
+  description   = "Prod GPU launch template"
+  ebs_optimized = true
+  iam_instance_profile {
+    arn = aws_iam_instance_profile.CompilerExplorerRole.arn
+  }
+  image_id               = local.gpu_image_id
+  user_data              = local.gpu_user_data
+  key_name               = "mattgodbolt"
+  vpc_security_group_ids = [aws_security_group.CompilerExplorer.id]
+  instance_type          = "g4dn.xlarge"
+
+  tag_specifications {
+    resource_type = "volume"
+
+    tags = {
+      Site        = "CompilerExplorer"
+      Environment = "GPU"
+    }
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+
+    tags = {
+      Site        = "CompilerExplorer"
+      Environment = "GPU"
+      Name        = "GPU"
+    }
+  }
+}
+
 resource "aws_launch_template" "CompilerExplorer-prod" {
   name          = "ce-prod"
   description   = "Production launch template"
diff --git a/terraform/security.tf b/terraform/security.tf
index 7c06efb4c..7446200fe 100644
--- a/terraform/security.tf
+++ b/terraform/security.tf
@@ -52,6 +52,7 @@ resource "aws_security_group_rule" "CE_ConanHttpFromAlb" {
   description              = "Allow HTTP access from the ALB"
 }
 
+// TODO remove these
 resource "aws_security_group_rule" "CE_GPUHttpFromAlb" {
   security_group_id        = aws_security_group.CompilerExplorer.id
   type                     = "ingress"
@@ -250,7 +251,7 @@ data "aws_iam_policy_document" "CeModifyStoredState" {
       "dynamodb:Scan",
       "dynamodb:Query"
     ]
-    resources = [aws_dynamodb_table.links.arn,aws_dynamodb_table.gpulinks.arn]
+    resources = [aws_dynamodb_table.links.arn]
   }
   statement {
     sid       = "S3AccessSid"
diff --git a/terraform/tg.tf b/terraform/tg.tf
index aa6dca6c5..3956da2d9 100644
--- a/terraform/tg.tf
+++ b/terraform/tg.tf
@@ -4,6 +4,7 @@ variable "ce-target-groups" {
     "prod"    = 1
     "staging" = 2
     "beta"    = 3
+    "gpu"     = 4
   }
 }
 
@@ -56,6 +57,7 @@ resource "aws_alb_target_group_attachment" "CEConanServerTargetInstance" {
   port             = 1080
 }
 
+// TODO remove me
 resource "aws_alb_target_group" "gpu" {
   lifecycle {
     create_before_destroy = true

From 9ce073796dd18ded5b5d4665dc9e3b9953d34ae7 Mon Sep 17 00:00:00 2001
From: Daniel Thornburgh <mysterymath@gmail.com>
Date: Mon, 14 Nov 2022 19:43:19 -0800
Subject: [PATCH 091/161] Add llvm-mos trunk. (#874)

* Add llvm-mos trunk.

This fetches the llvm-mos 6502 toolchain and SDK.

See http://llvm-mos.org and https://github.com/llvm-mos/llvm-mos-sdk for
details.

* Use a nightly build for llvm-mos.
---
 bin/yaml/c.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/bin/yaml/c.yaml b/bin/yaml/c.yaml
index fb584f0e8..2c87b9379 100644
--- a/bin/yaml/c.yaml
+++ b/bin/yaml/c.yaml
@@ -28,6 +28,16 @@ compilers:
         - "2.17"
         - "2.18"
         - "2.19"
+    llvmmos:
+      if: nightly
+      type: nightlytarballs
+      compression: xz
+      dir: llvm-mos-{name}
+      check_exe: bin/clang --version
+      url: https://github.com/llvm-mos/llvm-mos-sdk/releases/latest/download/llvm-mos-linux.tar.xz
+      untar_dir: llvm-mos
+      targets:
+        - trunk
     gcc:
       type: s3tarballs
       check_exe: bin/gcc -v

From 32d55d14592b7ef79787d8e2a717468238bacd9f Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 21:54:30 -0600
Subject: [PATCH 092/161] zomg bash

---
 init/start.sh | 2 +-
 site-gpu.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/init/start.sh b/init/start.sh
index 4362014d6..4eb92bddd 100755
--- a/init/start.sh
+++ b/init/start.sh
@@ -8,7 +8,7 @@ CE_USER=ce
 DEPLOY_DIR=${PWD}/.deploy
 COMPILERS_ARG=
 COMPILERS_FILE=$DEPLOY_DIR/discovered-compilers.json
-CE_PROP_ENV=--env amazon
+CE_PROP_ENV="--env amazon"
 
 echo Running in environment "${ENV}"
 # shellcheck disable=SC1090
diff --git a/site-gpu.sh b/site-gpu.sh
index 372176c97..805de0d70 100644
--- a/site-gpu.sh
+++ b/site-gpu.sh
@@ -2,5 +2,5 @@
 # Sourced when running in gpu mode
 export BRANCH=gpu
 export NODE_ENV=production
-export CE_PROP_ENV=${CE_PROP_ENV} --env gpu
+export CE_PROP_ENV="${CE_PROP_ENV} --env gpu"
 /infra/setup-gpu-node-devices.sh

From 6e8bf4ca31a928bad1a861a6cd6c6cea31df8b07 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 14 Nov 2022 23:06:49 -0600
Subject: [PATCH 093/161] As terraformed

---
 terraform/lc.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/lc.tf b/terraform/lc.tf
index 4341afd9e..96f755d3f 100644
--- a/terraform/lc.tf
+++ b/terraform/lc.tf
@@ -2,7 +2,7 @@ locals {
   image_id          = "ami-08b5bae0de035991b"
   staging_image_id  = "ami-08b5bae0de035991b"
   beta_image_id     = "ami-08b5bae0de035991b"
-  gpu_image_id      = "ami-0ea161f7cd222e436"
+  gpu_image_id      = "ami-05b58707118bd281d"
   staging_user_data = base64encode("staging")
   beta_user_data    = base64encode("beta")
   gpu_user_data     = base64encode("gpu")

From bc1cb932360e2f796ca13fe52f38465f8ca30814 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 15 Nov 2022 07:39:11 -0600
Subject: [PATCH 094/161] Remove older resources

---
 terraform/alb.tf            | 23 ----------
 terraform/cloudfront_gpu.tf | 87 -------------------------------------
 terraform/ec2.tf            | 34 ---------------
 terraform/security.tf       | 32 --------------
 terraform/tg.tf             | 26 -----------
 5 files changed, 202 deletions(-)
 delete mode 100644 terraform/cloudfront_gpu.tf

diff --git a/terraform/alb.tf b/terraform/alb.tf
index 9b73daa41..9575eda60 100644
--- a/terraform/alb.tf
+++ b/terraform/alb.tf
@@ -201,26 +201,3 @@ resource "aws_alb_listener_rule" "compiler-explorer-alb-listen-https-stats" {
   }
   listener_arn = aws_alb_listener.compiler-explorer-alb-listen-https.arn
 }
-
-resource "aws_alb_listener" "gpu-alb-listen-http" {
-  default_action {
-    type             = "forward"
-    target_group_arn = aws_alb_target_group.gpu.arn
-  }
-
-  load_balancer_arn = aws_alb.GccExplorerApp.arn
-  port              = 1081
-  protocol          = "HTTP"
-}
-
-resource "aws_alb_listener" "gpu-alb-listen-https" {
-  default_action {
-    type             = "forward"
-    target_group_arn = aws_alb_target_group.gpu.arn
-  }
-  load_balancer_arn = aws_alb.GccExplorerApp.arn
-  port              = 1444
-  protocol          = "HTTPS"
-  ssl_policy        = "ELBSecurityPolicy-2015-05"
-  certificate_arn   = data.aws_acm_certificate.godbolt-org-et-al.arn
-}
diff --git a/terraform/cloudfront_gpu.tf b/terraform/cloudfront_gpu.tf
deleted file mode 100644
index d9f52476f..000000000
--- a/terraform/cloudfront_gpu.tf
+++ /dev/null
@@ -1,87 +0,0 @@
-// TODO delete me
-resource "aws_cloudfront_distribution" "gpu-compiler-explorer-com" {
-  origin {
-    domain_name = "compiler-explorer.s3.amazonaws.com"
-    origin_id   = "S3-compiler-explorer"
-  }
-  origin {
-    domain_name = aws_alb.GccExplorerApp.dns_name
-    origin_id   = "GccExplorerApp"
-    custom_origin_config {
-      http_port                = 1081
-      https_port               = 1444
-      origin_read_timeout      = 60
-      origin_keepalive_timeout = 60
-      origin_protocol_policy   = "https-only"
-      origin_ssl_protocols     = [
-        "TLSv1",
-        "TLSv1.2",
-        "TLSv1.1"
-      ]
-    }
-  }
-
-  enabled          = true
-  is_ipv6_enabled  = true
-  retain_on_delete = true
-  aliases          = [
-    "gpu.compiler-explorer.com"
-  ]
-
-  viewer_certificate {
-    acm_certificate_arn      = data.aws_acm_certificate.godbolt-org-et-al.arn
-    ssl_support_method       = "sni-only"
-    minimum_protocol_version = "TLSv1.1_2016"
-  }
-
-  logging_config {
-    include_cookies = false
-    bucket          = "compiler-explorer-logs.s3.amazonaws.com"
-    prefix          = "cloudfront/"
-  }
-
-  http_version = "http2"
-
-  restrictions {
-    geo_restriction {
-      restriction_type = "blacklist"
-      locations        = [
-        "CU",
-        "IR",
-        "KP",
-        "SD",
-        "SY"
-      ]
-    }
-  }
-
-  default_cache_behavior {
-    allowed_methods = [
-      "HEAD",
-      "DELETE",
-      "POST",
-      "GET",
-      "OPTIONS",
-      "PUT",
-      "PATCH"
-    ]
-    cached_methods = [
-      "HEAD",
-      "GET"
-    ]
-    forwarded_values {
-      cookies {
-        forward = "all"
-      }
-      query_string = true
-      headers      = [
-        "Accept",
-        "Host",
-        "Authorization"
-      ]
-    }
-    target_origin_id       = "GccExplorerApp"
-    viewer_protocol_policy = "redirect-to-https"
-    compress               = true
-  }
-}
diff --git a/terraform/ec2.tf b/terraform/ec2.tf
index 1a59d3698..522b45be1 100644
--- a/terraform/ec2.tf
+++ b/terraform/ec2.tf
@@ -2,7 +2,6 @@ locals {
   runner_image_id  = "ami-0cc6fd5f52bd05b88"
   conan_image_id   = "ami-0b41dc7a318b530bd"
   builder_image_id = "ami-0ef4921e9d82c03fb"
-  gpu_image_id_old     = "ami-01d05f46623827076"
   admin_subnet     = module.ce_network.subnet["1a"].id
 }
 
@@ -130,36 +129,3 @@ resource "aws_instance" "CERunner" {
     Name = "CERunner"
   }
 }
-
-// TODO remove this
-resource "aws_instance" "GPUNode" {
-  ami                         = local.gpu_image_id_old
-  iam_instance_profile        = aws_iam_instance_profile.CompilerExplorerRole.name
-  ebs_optimized               = false
-  instance_type               = "g4dn.xlarge"
-  monitoring                  = false
-  key_name                    = "mattgodbolt"
-  subnet_id                   = local.admin_subnet
-  vpc_security_group_ids      = [aws_security_group.CompilerExplorer.id]
-  associate_public_ip_address = true
-  source_dest_check           = false
-  user_data                   = "gpu"
-
-  lifecycle {
-    ignore_changes = [
-      // Seemingly needed to not replace stopped instances
-      associate_public_ip_address
-    ]
-  }
-
-  root_block_device {
-    volume_type           = "gp2"
-    volume_size           = 24
-    delete_on_termination = false
-  }
-
-  tags = {
-    Site        = "CompilerExplorer"
-    Name        = "GPUNode"
-  }
-}
diff --git a/terraform/security.tf b/terraform/security.tf
index 7446200fe..1e29f6cae 100644
--- a/terraform/security.tf
+++ b/terraform/security.tf
@@ -52,17 +52,6 @@ resource "aws_security_group_rule" "CE_ConanHttpFromAlb" {
   description              = "Allow HTTP access from the ALB"
 }
 
-// TODO remove these
-resource "aws_security_group_rule" "CE_GPUHttpFromAlb" {
-  security_group_id        = aws_security_group.CompilerExplorer.id
-  type                     = "ingress"
-  from_port                = 1081
-  to_port                  = 1081
-  source_security_group_id = aws_security_group.CompilerExplorerAlb.id
-  protocol                 = "tcp"
-  description              = "Allow HTTP access from the ALB"
-}
-
 resource "aws_security_group_rule" "CE_HttpsFromAlb" {
   security_group_id        = aws_security_group.CompilerExplorer.id
   type                     = "ingress"
@@ -83,16 +72,6 @@ resource "aws_security_group_rule" "CE_ConanHttpsFromAlb" {
   description              = "Allow HTTPS access from the ALB"
 }
 
-resource "aws_security_group_rule" "CE_GPUHttpsFromAlb" {
-  security_group_id        = aws_security_group.CompilerExplorer.id
-  type                     = "ingress"
-  from_port                = 1444
-  to_port                  = 1444
-  source_security_group_id = aws_security_group.CompilerExplorerAlb.id
-  protocol                 = "tcp"
-  description              = "Allow HTTPS access from the ALB"
-}
-
 resource "aws_security_group" "CompilerExplorerAlb" {
   vpc_id      = module.ce_network.vpc.id
   name        = "ce-alb-sg"
@@ -124,17 +103,6 @@ resource "aws_security_group_rule" "ALB_ConanHttpsFromAnywhere" {
   description       = "Allow HTTPS access from anywhere"
 }
 
-resource "aws_security_group_rule" "ALB_GPUHttpsFromAnywhere" {
-  security_group_id = aws_security_group.CompilerExplorerAlb.id
-  type              = "ingress"
-  from_port         = 1444
-  to_port           = 1444
-  cidr_blocks       = ["0.0.0.0/0"]
-  ipv6_cidr_blocks  = ["::/0"]
-  protocol          = "tcp"
-  description       = "Allow HTTPS access from anywhere"
-}
-
 resource "aws_security_group_rule" "ALB_EgressToAnywhere" {
   security_group_id = aws_security_group.CompilerExplorerAlb.id
   type              = "egress"
diff --git a/terraform/tg.tf b/terraform/tg.tf
index 3956da2d9..ba804f18e 100644
--- a/terraform/tg.tf
+++ b/terraform/tg.tf
@@ -56,29 +56,3 @@ resource "aws_alb_target_group_attachment" "CEConanServerTargetInstance" {
   target_id        = aws_instance.ConanNode.id
   port             = 1080
 }
-
-// TODO remove me
-resource "aws_alb_target_group" "gpu" {
-  lifecycle {
-    create_before_destroy = true
-  }
-  name                 = "GPUGroup"
-  port                 = 1081
-  protocol             = "HTTP"
-  vpc_id               = module.ce_network.vpc.id
-  deregistration_delay = 15
-  health_check {
-    path                = "/healthcheck"
-    timeout             = 3
-    unhealthy_threshold = 3
-    healthy_threshold   = 2
-    interval            = 5
-    protocol            = "HTTP"
-  }
-}
-
-resource "aws_alb_target_group_attachment" "CEGPUServerTargetInstance" {
-  target_group_arn = aws_alb_target_group.gpu.id
-  target_id        = aws_instance.GPUNode.id
-  port             = 1081
-}

From 7156e7c08908c1c8b542ab73b38ef93a94ab9074 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 15 Nov 2022 07:47:39 -0600
Subject: [PATCH 095/161] Spot pricing for GPU

---
 terraform/asg.tf | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/terraform/asg.tf b/terraform/asg.tf
index 789ae0cf2..caa0e8c14 100644
--- a/terraform/asg.tf
+++ b/terraform/asg.tf
@@ -137,14 +137,28 @@ resource "aws_autoscaling_group" "gpu" {
   default_cooldown          = local.cooldown
   health_check_grace_period = local.grace_period
   health_check_type         = "ELB"
-  launch_template {
-    id      = aws_launch_template.CompilerExplorer-prod-gpu.id
-    version = "$Latest"
-  }
   max_size            = 2
   min_size            = 0
   name                = "gpu"
   vpc_zone_identifier = local.subnets
 
+  mixed_instances_policy {
+    instances_distribution {
+      on_demand_allocation_strategy            = "prioritized"
+      // This base value is zero so we don't have any non-spot instances. We may wish to bump this if we have issues
+      // getting spot capacity.
+      on_demand_base_capacity                  = 0
+      on_demand_percentage_above_base_capacity = 0
+      spot_allocation_strategy                 = "lowest-price"
+      spot_instance_pools                      = 2
+    }
+    launch_template {
+      launch_template_specification {
+        launch_template_id = aws_launch_template.CompilerExplorer-prod-gpu.id
+        version            = "$Latest"
+      }
+    }
+  }
+
   target_group_arns = [aws_alb_target_group.ce["gpu"].arn]
 }

From 69d8e88cfe9a0c82e680a39e45947dc8db575ce9 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 15 Nov 2022 19:31:54 -0600
Subject: [PATCH 096/161] autoscale the GPUses

---
 terraform/asg.tf | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/terraform/asg.tf b/terraform/asg.tf
index caa0e8c14..b1fe2f91c 100644
--- a/terraform/asg.tf
+++ b/terraform/asg.tf
@@ -160,5 +160,39 @@ resource "aws_autoscaling_group" "gpu" {
     }
   }
 
+  enabled_metrics = [
+    "GroupDesiredCapacity",
+    "GroupInServiceCapacity",
+    "GroupInServiceInstances",
+    "GroupMaxSize",
+    "GroupMinSize",
+    "GroupPendingCapacity",
+    "GroupPendingInstances",
+    "GroupStandbyCapacity",
+    "GroupStandbyInstances",
+    "GroupTerminatingCapacity",
+    "GroupTerminatingInstances",
+    "GroupTotalCapacity",
+    "GroupTotalInstances",
+  ]
+
   target_group_arns = [aws_alb_target_group.ce["gpu"].arn]
 }
+
+
+resource "aws_autoscaling_policy" "gpu" {
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  autoscaling_group_name    = aws_autoscaling_group.gpu.name
+  name                      = "cpu-tracker"
+  policy_type               = "TargetTrackingScaling"
+  estimated_instance_warmup = local.grace_period + 30
+  target_tracking_configuration {
+    predefined_metric_specification {
+      predefined_metric_type = "ASGAverageCPUUtilization"
+    }
+    target_value = 75.0
+  }
+}

From f8edc1a0739044dbe57b55e665c1838e0ee37e25 Mon Sep 17 00:00:00 2001
From: Paul Taylor <paul.e.taylor@me.com>
Date: Thu, 17 Nov 2022 10:27:42 -0800
Subject: [PATCH 097/161] add nvhpc-sdk 22.11 (#877)

---
 bin/yaml/cpp.yaml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 60368899f..2c1e55b8c 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -776,15 +776,12 @@ compilers:
         - name: x86_64_22_7
           arch: x86_64
           version: "22.7"
-#        - name: aarch64_22_7
-#          arch: aarch64
-#          version: "22.7"
         - name: x86_64_22_9
           arch: x86_64
           version: "22.9"
-#        - name: aarch64_22_9
-#          arch: aarch64
-#          version: "22.9"
+        - name: x86_64_22_11
+          arch: x86_64
+          version: "22.11"
     nightly:
       if: nightly
       gcc:

From 92f73e995a7583270c11755ecfd4b1c17f237e7c Mon Sep 17 00:00:00 2001
From: Quinton Miller <nicetas.c@gmail.com>
Date: Fri, 18 Nov 2022 13:57:54 +0800
Subject: [PATCH 098/161] Update Crystal 1.6.1 -> 1.6.2 (#879)

---
 bin/yaml/crystal.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/yaml/crystal.yaml b/bin/yaml/crystal.yaml
index ad3fe5543..db7ed264e 100644
--- a/bin/yaml/crystal.yaml
+++ b/bin/yaml/crystal.yaml
@@ -15,7 +15,7 @@ compilers:
       - ln -s /opt/compiler-explorer/libs/libpcre/8.45/x86_64/lib/libpcre.a # for 1.1.1 and before
       - ln -s /opt/compiler-explorer/libs/libevent/2.1.12/x86_64/lib/libevent.a
     targets:
-      - 1.6.1
+      - 1.6.2
       - 1.5.1
       - 1.4.1
       - 1.3.2

From 693ba2c71deeef847e9eb9c8327bdf6745c21dbd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= <ruben@rinconblanco.es>
Date: Fri, 18 Nov 2022 06:59:11 +0100
Subject: [PATCH 099/161] Wait for 3.11 until we install it

---
 .github/workflows/python-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
index e248df441..e80916d04 100644
--- a/.github/workflows/python-tests.yaml
+++ b/.github/workflows/python-tests.yaml
@@ -10,7 +10,7 @@ jobs:
         python-version:
           - 3.9
           - '3.10'
-          - 3.11
+#         - 3.11
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}

From e85769fbbb872c3a239c0e839c8cde7ccbff8893 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Fri, 18 Nov 2022 22:58:24 +0100
Subject: [PATCH 100/161] Fix hpcsdk download

---
 bin/yaml/cpp.yaml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 2c1e55b8c..993964476 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -744,8 +744,6 @@ compilers:
       dir: hpc_sdk/Linux_{arch}/{version}
       check_stderr_on_stdout: true
       check_exe: compilers/bin/nvc++ --version
-      fetch:
-        - https://developer.download.nvidia.com/hpc-sdk/{version}/nvhpc_2022_{{version|replace(".","")}}_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}_{arch}.tar.gz
       script: |
         mkdir -p "nvhpc_sdk_{version}_{arch}";
         tar -f "nvhpc_sdk_{version}_{arch}.tar.gz" -C "nvhpc_sdk_{version}_{arch}" --wildcards --strip-components=1 -pzx "nvhpc_*/*"
@@ -776,12 +774,21 @@ compilers:
         - name: x86_64_22_7
           arch: x86_64
           version: "22.7"
+          dot_removed_version: "227"
+          fetch:
+            - https://developer.download.nvidia.com/hpc-sdk/{version}/nvhpc_2022_{dot_removed_version}_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}_{arch}.tar.gz
         - name: x86_64_22_9
           arch: x86_64
           version: "22.9"
+          dot_removed_version: "229"
+          fetch:
+            - https://developer.download.nvidia.com/hpc-sdk/{version}/nvhpc_2022_{dot_removed_version}_Linux_{arch}_cuda_11.7.tar.gz nvhpc_sdk_{version}_{arch}.tar.gz
         - name: x86_64_22_11
           arch: x86_64
           version: "22.11"
+          dot_removed_version: "2211"
+          fetch:
+            - https://developer.download.nvidia.com/hpc-sdk/{version}/nvhpc_2022_{dot_removed_version}_Linux_{arch}_cuda_11.8.tar.gz nvhpc_sdk_{version}_{arch}.tar.gz
     nightly:
       if: nightly
       gcc:

From ed04c1a05f6fd285bebec961e7d21d465f40c3d2 Mon Sep 17 00:00:00 2001
From: Partouf <partouf@gmail.com>
Date: Sat, 19 Nov 2022 16:10:35 +0100
Subject: [PATCH 101/161] add llvmmos compiler

---
 init/settings.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/init/settings.yml b/init/settings.yml
index 88055b8a2..3762c4caa 100644
--- a/init/settings.yml
+++ b/init/settings.yml
@@ -26,6 +26,9 @@ compiler:
     cshared:
         version: ANY
         libcxx: ANY
+    llvmmos:
+        version: ANY
+        libcxx: ANY
 
 build_type: [Debug]
 

From b1606801ce5985c3169bb9b65fc9fff18353ab80 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Sat, 19 Nov 2022 22:43:00 +0100
Subject: [PATCH 102/161] Update admin-daily-builds.sh

---
 admin-daily-builds.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/admin-daily-builds.sh b/admin-daily-builds.sh
index 2460995d8..dc1ecfa8b 100755
--- a/admin-daily-builds.sh
+++ b/admin-daily-builds.sh
@@ -42,7 +42,7 @@ build_cpp_libraries() {
         -v/opt:/opt:ro \
         -e "CONAN_PASSWORD=${CONAN_PASSWORD}" \
         "compilerexplorer/library-builder" \
-        bash "${COMMAND}" "c++" "all" "all"
+        bash "${COMMAND}" "c++" "all" "all" || true
 }
 
 build_rust_libraries() {
@@ -57,7 +57,7 @@ build_rust_libraries() {
         -v/opt:/opt:ro \
         -e "CONAN_PASSWORD=${CONAN_PASSWORD}" \
         "compilerexplorer/library-builder" \
-        bash "${COMMAND}" "rust" "all" "all"
+        bash "${COMMAND}" "rust" "all" "all" || true
 }
 
 init_logspout

From 5503cfb34b6d9acf0b3c0cb29bf6ab58674073e1 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Sun, 20 Nov 2022 15:51:42 +0100
Subject: [PATCH 103/161] move cmake install to yaml (#871)

---
 bin/lib/installation_context.py      |  1 +
 bin/yaml/tools.yaml                  | 11 +++++++++++
 update_compilers/install_binaries.sh | 18 ------------------
 3 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/bin/lib/installation_context.py b/bin/lib/installation_context.py
index 5bb7d9e6b..3ac1d02ca 100644
--- a/bin/lib/installation_context.py
+++ b/bin/lib/installation_context.py
@@ -173,6 +173,7 @@ def remove_dir(self, directory: Union[str, Path]) -> None:
             _LOGGER.info("Removing %s", directory)
 
     def check_link(self, source: str, link: str) -> bool:
+        _LOGGER.debug("check link %s", link)
         try:
             link = self.read_link(link)
             _LOGGER.debug("readlink returned %s", link)
diff --git a/bin/yaml/tools.yaml b/bin/yaml/tools.yaml
index 0303bf3a2..12fb58795 100644
--- a/bin/yaml/tools.yaml
+++ b/bin/yaml/tools.yaml
@@ -1,4 +1,15 @@
 tools:
+  cmake:
+    type: tarballs
+    compression: gz
+    url: https://github.com/Kitware/CMake/releases/download/v{name}/cmake-{name}-Linux-x86_64.tar.gz
+    strip_components: 1
+    dir: cmake-v{name}
+    create_untar_dir: true
+    check_exe: bin/cmake --help
+    symlink: cmake
+    targets:
+      - 3.24.3
   pvs-studio:
     if:
       - non-free
diff --git a/update_compilers/install_binaries.sh b/update_compilers/install_binaries.sh
index e56460b8b..ca6e67c95 100755
--- a/update_compilers/install_binaries.sh
+++ b/update_compilers/install_binaries.sh
@@ -6,24 +6,6 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=common.inc
 . "${DIR}/common.inc"
 
-#########################
-# cmake
-
-if [[ ! -x ${OPT}/cmake-v3.18.2/bin/cmake ]]; then
-    cd "${OPT}"
-    mkdir cmake-v3.18.2
-    fetch https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-Linux-x86_64.tar.gz | tar zxf - --strip-components 1 -C cmake-v3.18.2
-fi
-
-if [[ ! -x ${OPT}/cmake-v3.23.1/bin/cmake ]]; then
-    cd "${OPT}"
-    mkdir cmake-v3.23.1
-    fetch https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-Linux-x86_64.tar.gz | tar zxf - --strip-components 1 -C cmake-v3.23.1
-fi
-
-rm -Rf "${OPT}/cmake"
-ln -s "${OPT}/cmake-v3.23.1" "${OPT}/cmake"
-
 #########################
 # pahole
 

From af67bdd719acddc53a1be306237664bedb55125f Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Sun, 20 Nov 2022 15:52:56 +0100
Subject: [PATCH 104/161] add ninja to yaml (#872)

---
 bin/lib/installable/archives.py    |  8 ++++++--
 bin/lib/installable/installable.py |  1 +
 bin/yaml/tools.yaml                | 11 +++++++++++
 init/start.sh                      |  6 ++++++
 4 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/bin/lib/installable/archives.py b/bin/lib/installable/archives.py
index 90ea78d4e..705381eeb 100644
--- a/bin/lib/installable/archives.py
+++ b/bin/lib/installable/archives.py
@@ -141,7 +141,6 @@ class TarballInstallable(Installable):
     def __init__(self, install_context: InstallationContext, config: Dict[str, Any]):
         super().__init__(install_context, config)
         self.install_path = self.config_get("dir")
-        self.install_path_symlink = self.config_get("symlink", False)
         self.untar_path = self.config_get("untar_dir", self.install_path)
         if self.config_get("create_untar_dir", False):
             self.untar_to = self.untar_path
@@ -245,6 +244,8 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.configure_command = command_config(self.config_get("configure_command", []))
         self.strip = self.config_get("strip", False)
         self._setup_check_exe(self.install_path)
+        if self.install_path_symlink:
+            self._setup_check_link(self.install_path, self.install_path_symlink)
 
     def stage(self, staging: StagingDir) -> None:
         # Unzip does not support stdin piping so we need to create a file
@@ -254,7 +255,8 @@ def stage(self, staging: StagingDir) -> None:
             if self.extract_into_folder:
                 unzip_cmd.extend(["-d", self.folder_to_rename])
             self.install_context.stage_command(staging, unzip_cmd)
-            self.install_context.stage_command(staging, ["mv", self.folder_to_rename, self.install_path])
+            if self.folder_to_rename != self.install_path:
+                self.install_context.stage_command(staging, ["mv", self.folder_to_rename, self.install_path])
         if self.configure_command:
             self.install_context.stage_command(staging, self.configure_command)
         if self.strip:
@@ -276,6 +278,8 @@ def install(self) -> None:
         with self.install_context.new_staging_dir() as staging:
             self.stage(staging)
             self.install_context.move_from_staging(staging, self.install_path)
+            if self.install_path_symlink:
+                self.install_context.set_link(Path(self.install_path), self.install_path_symlink)
 
     def __repr__(self) -> str:
         return f"ZipArchiveInstallable({self.name}, {self.install_path})"
diff --git a/bin/lib/installable/installable.py b/bin/lib/installable/installable.py
index 1c6a2f690..36b954226 100644
--- a/bin/lib/installable/installable.py
+++ b/bin/lib/installable/installable.py
@@ -49,6 +49,7 @@ def __init__(self, install_context: InstallationContext, config: Dict[str, Any])
         self.install_path = ""
         self.after_stage_script = self.config_get("after_stage_script", [])
         self._logger = logging.getLogger(self.name)
+        self.install_path_symlink = self.config_get("symlink", False)
 
     def to_json_dict(self) -> Dict[str, Any]:
         return {key: value for key, value in self.__dict__.items() if isinstance(value, SimpleJsonType)}
diff --git a/bin/yaml/tools.yaml b/bin/yaml/tools.yaml
index 12fb58795..7fa90f9b7 100644
--- a/bin/yaml/tools.yaml
+++ b/bin/yaml/tools.yaml
@@ -10,6 +10,17 @@ tools:
     symlink: cmake
     targets:
       - 3.24.3
+  ninja:
+    type: ziparchive
+    extract_into_folder: true
+    folder: ninja-v{name}
+    url: https://github.com/ninja-build/ninja/releases/download/v{name}/ninja-linux.zip
+    dir: ninja-v{name}
+    create_untar_dir: true
+    check_exe: ninja --version
+    symlink: ninja
+    targets:
+      - 1.11.1
   pvs-studio:
     if:
       - non-free
diff --git a/init/start.sh b/init/start.sh
index 4eb92bddd..91c215dfe 100755
--- a/init/start.sh
+++ b/init/start.sh
@@ -87,6 +87,11 @@ install_asmparser() {
     cp /opt/compiler-explorer/asm-parser/asm-parser /usr/local/bin
 }
 
+install_ninja() {
+    rm -f /usr/local/bin/ninja
+    cp /opt/compiler-explorer/ninja/ninja /usr/local/bin
+}
+
 LOG_DEST_HOST=$(get_conf /compiler-explorer/logDestHost)
 LOG_DEST_PORT=$(get_conf /compiler-explorer/logDestPort)
 
@@ -96,6 +101,7 @@ cgcreate -a ${CE_USER}:${CE_USER} -g memory,pids,cpu,net_cls:ce-compile
 mount_opt
 update_code
 install_asmparser
+install_ninja
 
 cd "${DEPLOY_DIR}"
 

From 51fb2d6b4f3f50649c67cc43800f9f335a5f8ac6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 20 Nov 2022 15:53:47 +0100
Subject: [PATCH 105/161] Bump cryptography from 38.0.1 to 38.0.3 (#873)

Bumps [cryptography](https://github.com/pyca/cryptography) from 38.0.1 to 38.0.3.
- [Release notes](https://github.com/pyca/cryptography/releases)
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/38.0.1...38.0.3)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 poetry.lock | 54 ++++++++++++++++++++++++++---------------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index b4724c5f8..7eed1c908 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -198,7 +198,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 
 [[package]]
 name = "cryptography"
-version = "38.0.1"
+version = "38.0.3"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
 category = "main"
 optional = false
@@ -942,32 +942,32 @@ colorama = [
     {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"},
 ]
 cryptography = [
-    {file = "cryptography-38.0.1-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:10d1f29d6292fc95acb597bacefd5b9e812099d75a6469004fd38ba5471a977f"},
-    {file = "cryptography-38.0.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:3fc26e22840b77326a764ceb5f02ca2d342305fba08f002a8c1f139540cdfaad"},
-    {file = "cryptography-38.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:3b72c360427889b40f36dc214630e688c2fe03e16c162ef0aa41da7ab1455153"},
-    {file = "cryptography-38.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:194044c6b89a2f9f169df475cc167f6157eb9151cc69af8a2a163481d45cc407"},
-    {file = "cryptography-38.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca9f6784ea96b55ff41708b92c3f6aeaebde4c560308e5fbbd3173fbc466e94e"},
-    {file = "cryptography-38.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:16fa61e7481f4b77ef53991075de29fc5bacb582a1244046d2e8b4bb72ef66d0"},
-    {file = "cryptography-38.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d4ef6cc305394ed669d4d9eebf10d3a101059bdcf2669c366ec1d14e4fb227bd"},
-    {file = "cryptography-38.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3261725c0ef84e7592597606f6583385fed2a5ec3909f43bc475ade9729a41d6"},
-    {file = "cryptography-38.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:0297ffc478bdd237f5ca3a7dc96fc0d315670bfa099c04dc3a4a2172008a405a"},
-    {file = "cryptography-38.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:89ed49784ba88c221756ff4d4755dbc03b3c8d2c5103f6d6b4f83a0fb1e85294"},
-    {file = "cryptography-38.0.1-cp36-abi3-win32.whl", hash = "sha256:ac7e48f7e7261207d750fa7e55eac2d45f720027d5703cd9007e9b37bbb59ac0"},
-    {file = "cryptography-38.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ad7353f6ddf285aeadfaf79e5a6829110106ff8189391704c1d8801aa0bae45a"},
-    {file = "cryptography-38.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:896dd3a66959d3a5ddcfc140a53391f69ff1e8f25d93f0e2e7830c6de90ceb9d"},
-    {file = "cryptography-38.0.1-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:d3971e2749a723e9084dd507584e2a2761f78ad2c638aa31e80bc7a15c9db4f9"},
-    {file = "cryptography-38.0.1-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:79473cf8a5cbc471979bd9378c9f425384980fcf2ab6534b18ed7d0d9843987d"},
-    {file = "cryptography-38.0.1-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9e69ae01f99abe6ad646947bba8941e896cb3aa805be2597a0400e0764b5818"},
-    {file = "cryptography-38.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5067ee7f2bce36b11d0e334abcd1ccf8c541fc0bbdaf57cdd511fdee53e879b6"},
-    {file = "cryptography-38.0.1-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3e3a2599e640927089f932295a9a247fc40a5bdf69b0484532f530471a382750"},
-    {file = "cryptography-38.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c2e5856248a416767322c8668ef1845ad46ee62629266f84a8f007a317141013"},
-    {file = "cryptography-38.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:64760ba5331e3f1794d0bcaabc0d0c39e8c60bf67d09c93dc0e54189dfd7cfe5"},
-    {file = "cryptography-38.0.1-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b6c9b706316d7b5a137c35e14f4103e2115b088c412140fdbd5f87c73284df61"},
-    {file = "cryptography-38.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0163a849b6f315bf52815e238bc2b2346604413fa7c1601eea84bcddb5fb9ac"},
-    {file = "cryptography-38.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:d1a5bd52d684e49a36582193e0b89ff267704cd4025abefb9e26803adeb3e5fb"},
-    {file = "cryptography-38.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:765fa194a0f3372d83005ab83ab35d7c5526c4e22951e46059b8ac678b44fa5a"},
-    {file = "cryptography-38.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:52e7bee800ec869b4031093875279f1ff2ed12c1e2f74923e8f49c916afd1d3b"},
-    {file = "cryptography-38.0.1.tar.gz", hash = "sha256:1db3d807a14931fa317f96435695d9ec386be7b84b618cc61cfa5d08b0ae33d7"},
+    {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320"},
+    {file = "cryptography-38.0.3-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722"},
+    {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f"},
+    {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828"},
+    {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959"},
+    {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2"},
+    {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c"},
+    {file = "cryptography-38.0.3-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0"},
+    {file = "cryptography-38.0.3-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748"},
+    {file = "cryptography-38.0.3-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146"},
+    {file = "cryptography-38.0.3-cp36-abi3-win32.whl", hash = "sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0"},
+    {file = "cryptography-38.0.3-cp36-abi3-win_amd64.whl", hash = "sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220"},
+    {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd"},
+    {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55"},
+    {file = "cryptography-38.0.3-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b"},
+    {file = "cryptography-38.0.3-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36"},
+    {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d"},
+    {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7"},
+    {file = "cryptography-38.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249"},
+    {file = "cryptography-38.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50"},
+    {file = "cryptography-38.0.3-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0"},
+    {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8"},
+    {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436"},
+    {file = "cryptography-38.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548"},
+    {file = "cryptography-38.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a"},
+    {file = "cryptography-38.0.3.tar.gz", hash = "sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd"},
 ]
 dill = [
     {file = "dill-0.3.5.1-py2.py3-none-any.whl", hash = "sha256:33501d03270bbe410c72639b350e941882a8b0fd55357580fbc873fba0c59302"},

From 44ab97b61e0313f87c8c5dd299c98c897514b26c Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 20 Nov 2022 11:08:42 -0600
Subject: [PATCH 106/161] Keep old gpu builds too (#881)

Also slight rephrasing of the environments to hopefully avoid forgetting this kind of things next time.
---
 bin/lib/amazon.py     |  2 +-
 bin/lib/cli/runner.py |  8 +++++---
 bin/lib/env.py        | 15 ++++++---------
 3 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/bin/lib/amazon.py b/bin/lib/amazon.py
index a83b402dc..1aaf446d9 100644
--- a/bin/lib/amazon.py
+++ b/bin/lib/amazon.py
@@ -184,7 +184,7 @@ def get_current_key(cfg: Config) -> Optional[str]:
 
 def get_all_current() -> List[str]:
     versions = []
-    for branch in ["release", "beta", "staging"]:
+    for branch in [env.value for env in Environment if env.keep_builds]:
         try:
             o = s3_client.get_object(Bucket="compiler-explorer", Key="version/{}".format(branch))
             versions.append(o["Body"].read().decode("utf-8").strip())
diff --git a/bin/lib/cli/runner.py b/bin/lib/cli/runner.py
index 1b7a196cb..aeb508702 100644
--- a/bin/lib/cli/runner.py
+++ b/bin/lib/cli/runner.py
@@ -5,7 +5,7 @@
 import click
 import boto3
 import botocore.exceptions
-from lib.env import EnvironmentNoProd, EnvironmentNoRunner
+from lib.env import Environment
 
 from lib.instance import RunnerInstance
 from lib.ssh import get_remote_file, run_remote_shell, exec_remote, exec_remote_to_stdout
@@ -58,7 +58,9 @@ def _s3_key_for(environment, version):
 
 
 @runner.command(name="uploaddiscovery")
-@click.argument("environment", required=True, type=click.Choice([env.value for env in EnvironmentNoRunner]))
+@click.argument(
+    "environment", required=True, type=click.Choice([env.value for env in Environment if env != Environment.RUNNER])
+)
 @click.argument("version", required=True)
 def runner_uploaddiscovery(environment: str, version: str):
     """Execute compiler discovery on the builder instance."""
@@ -82,7 +84,7 @@ def runner_discoveryexists(environment: str, version: str):
 
 
 @runner.command(name="safeforprod")
-@click.argument("environment", required=True, type=click.Choice([env.value for env in EnvironmentNoProd]))
+@click.argument("environment", required=True, type=click.Choice([env.value for env in Environment if not env.is_prod]))
 @click.argument("version", required=True)
 def runner_safeforprod(environment: str, version: str):
     """Mark discovery file as safe to use on production."""
diff --git a/bin/lib/env.py b/bin/lib/env.py
index 4dce7bc66..7806ad810 100644
--- a/bin/lib/env.py
+++ b/bin/lib/env.py
@@ -9,16 +9,13 @@ class Environment(Enum):
     GPU = "gpu"
     RUNNER = "runner"
 
+    @property
+    def keep_builds(self):
+        return self in (Environment.PROD, Environment.BETA, Environment.STAGING, Environment.GPU)
 
-class EnvironmentNoRunner(Enum):
-    PROD = "prod"
-    BETA = "beta"
-    STAGING = "staging"
-
-
-class EnvironmentNoProd(Enum):
-    BETA = "beta"
-    STAGING = "staging"
+    @property
+    def is_prod(self):
+        return self in (Environment.PROD, Environment.GPU)
 
 
 @dataclass(frozen=True)

From f7aeb3410f767a891cce22e8ce57bb5fc39581ea Mon Sep 17 00:00:00 2001
From: TrellixVulnTeam <112716341+TrellixVulnTeam@users.noreply.github.com>
Date: Sun, 20 Nov 2022 11:29:34 -0600
Subject: [PATCH 107/161] Adding tarfile member sanitization to extractall()
 (#878)

---
 bin/lib/cdn.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/bin/lib/cdn.py b/bin/lib/cdn.py
index 472f7c682..a1886000a 100644
--- a/bin/lib/cdn.py
+++ b/bin/lib/cdn.py
@@ -90,7 +90,26 @@ def __unpack_tar(self):
         # unpack tar contents
         logger.debug('unpacking "%s" into "%s"', self.tar_file_path, self.tmpdir)
         with tarfile.open(self.tar_file_path) as tar:
-            tar.extractall(self.tmpdir)
+            def is_within_directory(directory, target):
+                
+                abs_directory = os.path.abspath(directory)
+                abs_target = os.path.abspath(target)
+            
+                prefix = os.path.commonprefix([abs_directory, abs_target])
+                
+                return prefix == abs_directory
+            
+            def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
+            
+                for member in tar.getmembers():
+                    member_path = os.path.join(path, member.name)
+                    if not is_within_directory(path, member_path):
+                        raise Exception("Attempted Path Traversal in Tar File")
+            
+                tar.extractall(path, members, numeric_owner=numeric_owner) 
+                
+            
+            safe_extract(tar, self.tmpdir)
 
         return list(get_directory_contents(self.tmpdir))
 

From 00a2df18426fd93d02962d17088575ad9b6529d3 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 20 Nov 2022 18:00:17 -0600
Subject: [PATCH 108/161] Fix lint

---
 bin/lib/cdn.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/bin/lib/cdn.py b/bin/lib/cdn.py
index a1886000a..d48828c17 100644
--- a/bin/lib/cdn.py
+++ b/bin/lib/cdn.py
@@ -90,25 +90,25 @@ def __unpack_tar(self):
         # unpack tar contents
         logger.debug('unpacking "%s" into "%s"', self.tar_file_path, self.tmpdir)
         with tarfile.open(self.tar_file_path) as tar:
+
             def is_within_directory(directory, target):
-                
+
                 abs_directory = os.path.abspath(directory)
                 abs_target = os.path.abspath(target)
-            
+
                 prefix = os.path.commonprefix([abs_directory, abs_target])
-                
+
                 return prefix == abs_directory
-            
+
             def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
-            
+
                 for member in tar.getmembers():
                     member_path = os.path.join(path, member.name)
                     if not is_within_directory(path, member_path):
                         raise Exception("Attempted Path Traversal in Tar File")
-            
-                tar.extractall(path, members, numeric_owner=numeric_owner) 
-                
-            
+
+                tar.extractall(path, members, numeric_owner=numeric_owner)
+
             safe_extract(tar, self.tmpdir)
 
         return list(get_directory_contents(self.tmpdir))

From 42ea267bb0c0dd582a59464a8ad076392d66b573 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sun, 20 Nov 2022 19:40:32 -0600
Subject: [PATCH 109/161] Dear me...budgetary updates

---
 terraform/cloudwatch.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/cloudwatch.tf b/terraform/cloudwatch.tf
index af3e50f1a..2afe2e329 100644
--- a/terraform/cloudwatch.tf
+++ b/terraform/cloudwatch.tf
@@ -3,8 +3,8 @@ data "aws_sns_topic" "alert" {
 }
 
 locals {
-  approx_monthly_budget = 750
-  alert_every           = 250
+  approx_monthly_budget = 1200
+  alert_every           = 200
 }
 
 resource "aws_cloudwatch_metric_alarm" "resp_90ile_15m_too_slow" {

From fff936a2186570e90969c31e24437e99ada68be4 Mon Sep 17 00:00:00 2001
From: amyspark <amy@amyspark.me>
Date: Wed, 23 Nov 2022 17:48:52 -0300
Subject: [PATCH 110/161] xsimd: Add a selection of 8.x and latest 9.x versions
 (#885)

---
 bin/yaml/libraries.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index d7017dda0..8b4685821 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -1686,6 +1686,10 @@ libraries:
       - 7.0.0
       - 7.6.0
       - 8.0.3
+      - 8.0.5
+      - 8.1.0
+      - 9.0.0
+      - 9.0.1
       type: github
     xtensor:
       check_file: README.md

From c260522b893a65066246963aad3f8173735a53c4 Mon Sep 17 00:00:00 2001
From: Christian Clauss <cclauss@me.com>
Date: Wed, 23 Nov 2022 22:23:20 +0100
Subject: [PATCH 111/161] Makefile: Add Python 3.11 (#876)

* Makefile: Add Python 3.11

https://docs.python.org/3.11/whatsnew/3.11.html

* Update python-tests.yaml

* Update releases.py
---
 .github/workflows/python-tests.yaml | 6 +++++-
 Makefile                            | 2 +-
 bin/lib/releases.py                 | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python-tests.yaml b/.github/workflows/python-tests.yaml
index e80916d04..ef95f0f1a 100644
--- a/.github/workflows/python-tests.yaml
+++ b/.github/workflows/python-tests.yaml
@@ -1,6 +1,10 @@
 name: Python tests
 
-on: [ push ]
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 jobs:
   build:
diff --git a/Makefile b/Makefile
index 7af84be70..ca05292f7 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@ export POETRY_HOME=$(CURDIR)/.poetry
 POETRY:=$(POETRY_HOME)/bin/poetry
 POETRY_VENVS=$(POETRY_HOME)/virtualenvs
 POETRY_DEPS:=$(POETRY_VENVS)/.deps
-SYS_PYTHON:=$(shell env PATH='/bin:/usr/bin:/usr/local/bin:$(PATH)' bash -c "command -v python3.10 || command -v python3.9 || echo .python-not-found")
+SYS_PYTHON:=$(shell env PATH='/bin:/usr/bin:/usr/local/bin:$(PATH)' bash -c "command -v python3.11 || command -v python3.10 || command -v python3.9 || echo .python-not-found")
 export PYTHONPATH=$(CURDIR)/bin
 
 .PHONY: help
diff --git a/bin/lib/releases.py b/bin/lib/releases.py
index 3beda23d9..029db35f8 100644
--- a/bin/lib/releases.py
+++ b/bin/lib/releases.py
@@ -13,7 +13,7 @@ def __str__(self) -> str:
 
 
 class VersionSource(Enum):
-    value: Tuple[int, str]
+    value: Tuple[int, str]  # type: ignore[assignment]
     TRAVIS = (0, "tr")
     GITHUB = (1, "gh")
 

From ea7af50072f8dabb29f386eaae1d62410890090d Mon Sep 17 00:00:00 2001
From: Robert Cohn <robert.s.cohn@intel.com>
Date: Wed, 23 Nov 2022 16:26:17 -0500
Subject: [PATCH 112/161] add oneapi 2022.2.0 2022.2.1 (#884)

---
 bin/yaml/cpp.yaml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 993964476..eaa62dbed 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -708,6 +708,14 @@ compilers:
           fetch:
             - https://registrationcenter-download.intel.com/akdlm/irc_nas/18717/l_dpcpp-cpp-compiler_p_{name}_offline.sh install.sh
           script: *intel-one-install-script
+        - name: 2022.2.0.8772
+          fetch:
+            - https://registrationcenter-download.intel.com/akdlm/irc_nas/18849/l_dpcpp-cpp-compiler_p_{name}_offline.sh install.sh
+          script: *intel-one-install-script
+        - name: 2022.2.1.16991
+          fetch:
+            - https://registrationcenter-download.intel.com/akdlm/irc_nas/19049/l_dpcpp-cpp-compiler_p_{name}_offline.sh install.sh
+          script: *intel-one-install-script
     intel-fortran:
       type: script
       dir: "intel-fortran-{name}"
@@ -739,6 +747,14 @@ compilers:
           fetch:
             - https://registrationcenter-download.intel.com/akdlm/irc_nas/18703/l_fortran-compiler_p_{name}_offline.sh install.sh
           script: *intel-one-install-script
+        - name: 2022.2.0.8773
+          fetch:
+            - https://registrationcenter-download.intel.com/akdlm/irc_nas/18909/l_fortran-compiler_p_{name}_offline.sh install.sh
+          script: *intel-one-install-script
+        - name: 2022.2.1.16992
+          fetch:
+            - https://registrationcenter-download.intel.com/akdlm/irc_nas/18998/l_fortran-compiler_p_{name}_offline.sh install.sh
+          script: *intel-one-install-script
     nvhpc-sdk:
       type: script
       dir: hpc_sdk/Linux_{arch}/{version}

From b7bada24b335d385baaea829d44fcf38b1b623e6 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Thu, 24 Nov 2022 07:07:22 -0600
Subject: [PATCH 113/161] Remove odin language

It's not used, and fails to install:

```
/opt/compiler-explorer/odin-dev-2022-08/odin: error while loading shared libraries: libLLVM-11.so.1: cannot open shared object file: No such file or directory
```

See also https://github.com/compiler-explorer/compiler-explorer/pull/4070
---
 bin/yaml/odin.yaml | 13 -------------
 1 file changed, 13 deletions(-)
 delete mode 100644 bin/yaml/odin.yaml

diff --git a/bin/yaml/odin.yaml b/bin/yaml/odin.yaml
deleted file mode 100644
index 63fd29b14..000000000
--- a/bin/yaml/odin.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-compilers:
-  odin:
-    type: ziparchive
-    check_exe: odin version
-    url: https://github.com/odin-lang/Odin/releases/download/{name}/odin-ubuntu-amd64-{name}.zip
-    dir: odin-{name}
-    extract_into_folder: true
-    folder: odin
-    after_stage_script:
-      - chmod 0755 odin
-    targets:
-      - dev-2022-09
-      - dev-2022-08

From 1f3845f78643c79d82a04528c4823bf5eb913a86 Mon Sep 17 00:00:00 2001
From: kasperk81 <83082615+kasperk81@users.noreply.github.com>
Date: Thu, 24 Nov 2022 15:21:46 +0200
Subject: [PATCH 114/161] Update dotnet.yaml (#869)

* Update dotnet.yaml
---
 bin/yaml/dotnet.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index ab194a5f3..ccdb618c0 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -9,6 +9,8 @@ compilers:
       - "/usr/bin/env DOTNET_CLI_HOME=/tmp /opt/compiler-explorer/{dir}/bin/coreclr/Linux.x64.Checked/.dotnet/dotnet --version"
     targets:
       - v6.0.1
+      - v6.0.11
+      - v7.0.0
   dotnet_nuget:
     type: s3tarballs
     dir: dotnet_nuget-{name}
@@ -19,3 +21,5 @@ compilers:
       - "cat /opt/compiler-explorer/{dir}/packages/microsoft.netcore.app.runtime.linux-x64/6.0.0/Microsoft.NETCore.App.versions.txt"
     targets:
       - v6.0.0
+      - v6.0.11
+      - v7.0.0

From efdec925ac74d31f6eebbada5a3f7755811a148d Mon Sep 17 00:00:00 2001
From: Steve <hez2010@outlook.com>
Date: Thu, 24 Nov 2022 23:52:20 +0900
Subject: [PATCH 115/161] Remove nuget dependency (#886)

---
 bin/yaml/dotnet.yaml | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index ccdb618c0..58b23765f 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -11,15 +11,3 @@ compilers:
       - v6.0.1
       - v6.0.11
       - v7.0.0
-  dotnet_nuget:
-    type: s3tarballs
-    dir: dotnet_nuget-{name}
-    untar_dir: nuget
-    check_exe:
-      - /bin/bash
-      - -c
-      - "cat /opt/compiler-explorer/{dir}/packages/microsoft.netcore.app.runtime.linux-x64/6.0.0/Microsoft.NETCore.App.versions.txt"
-    targets:
-      - v6.0.0
-      - v6.0.11
-      - v7.0.0

From de0b4bdb9d331ed0682f42a3f8501d82fbcd372c Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 25 Nov 2022 07:54:59 -0600
Subject: [PATCH 116/161] Add newer dot nets with their fix(es). Remove old
 dotnet, will be superseded

---
 bin/yaml/dotnet.yaml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index 58b23765f..2d0db27ae 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -2,12 +2,8 @@ compilers:
   dotnet:
     type: s3tarballs
     dir: dotnet-{name}
-    untar_dir: artifacts
-    check_exe:
-      - /bin/bash
-      - -c
-      - "/usr/bin/env DOTNET_CLI_HOME=/tmp /opt/compiler-explorer/{dir}/bin/coreclr/Linux.x64.Checked/.dotnet/dotnet --version"
+    untar_dir: Core_Root
+    check_exe: .dotnet/dotnet --version
     targets:
-      - v6.0.1
       - v6.0.11
       - v7.0.0

From 9a49bd49b072a59c452e308b384183f0d1883528 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 25 Nov 2022 08:48:30 -0600
Subject: [PATCH 117/161] zomg testing

---
 bin/yaml/dotnet.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index 2d0db27ae..59c7c8388 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -5,5 +5,7 @@ compilers:
     untar_dir: Core_Root
     check_exe: .dotnet/dotnet --version
     targets:
-      - v6.0.11
+      - name: v6.0.11
+        check_env:
+          - DOTNET_CLI_HOME=/tmp
       - v7.0.0

From df408d32aba3c67d5e38c4da6bef08c194efa133 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 25 Nov 2022 10:42:00 -0600
Subject: [PATCH 118/161] Hacky ci setup

---
 grafana/make-config-ci.sh |  8 ++++++++
 setup-ci.sh               | 10 ++++++++++
 setup-common.sh           |  2 +-
 3 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100755 grafana/make-config-ci.sh
 create mode 100755 setup-ci.sh

diff --git a/grafana/make-config-ci.sh b/grafana/make-config-ci.sh
new file mode 100755
index 000000000..20b229cf9
--- /dev/null
+++ b/grafana/make-config-ci.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+ENV=CI
+HOSTNAME=$(hostname)
+sed "s/@HOSTNAME@/${HOSTNAME}/g;s/@ENV@/${ENV}/g" /etc/grafana/agent.yaml.tpl > /etc/grafana/agent.yaml
+chmod 600 /etc/grafana/agent.yaml
diff --git a/setup-ci.sh b/setup-ci.sh
new file mode 100755
index 000000000..c9633ad6c
--- /dev/null
+++ b/setup-ci.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -exuo pipefail
+
+# NB this is run from the steps in (private) https://github.com/compiler-explorer/ce-ci
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "${DIR}"
+
+env EXTRA_NFS_ARGS="" MAKE_CONFIG_SUFFIX="-ci" "${DIR}/setup-common.sh"
diff --git a/setup-common.sh b/setup-common.sh
index f21faff18..a908be3a2 100755
--- a/setup-common.sh
+++ b/setup-common.sh
@@ -105,7 +105,7 @@ LOKI_PASSWORD=$(get_conf /compiler-explorer/lokiPassword)
 
 mkdir -p /etc/grafana
 cp /infra/grafana/agent.yaml /etc/grafana/agent.yaml.tpl
-cp /infra/grafana/make-config.sh /etc/grafana/make-config.sh
+cp "/infra/grafana/make-config${MAKE_CONFIG_SUFFIX:-}.sh" /etc/grafana/make-config.sh
 cp /infra/grafana/grafana-agent.service /lib/systemd/system/grafana-agent.service
 systemctl daemon-reload
 systemctl enable grafana-agent

From 92760c6f2835f14dee9536694d84280af8a2faf1 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 25 Nov 2022 10:58:28 -0600
Subject: [PATCH 119/161] Take two at ci

---
 setup-ci.sh     |  2 +-
 setup-common.sh | 31 ++++++++++++++++++++++---------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/setup-ci.sh b/setup-ci.sh
index c9633ad6c..3dfb3249c 100755
--- a/setup-ci.sh
+++ b/setup-ci.sh
@@ -7,4 +7,4 @@ set -exuo pipefail
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "${DIR}"
 
-env EXTRA_NFS_ARGS="" MAKE_CONFIG_SUFFIX="-ci" "${DIR}/setup-common.sh"
+env EXTRA_NFS_ARGS="" "${DIR}/setup-common.sh" ci
diff --git a/setup-common.sh b/setup-common.sh
index a908be3a2..db3c16e58 100755
--- a/setup-common.sh
+++ b/setup-common.sh
@@ -2,6 +2,9 @@
 
 set -exuo pipefail
 
+INSTALL_TYPE=${1:-non-ci}
+
+cock
 # https://askubuntu.com/questions/132059/how-to-make-a-package-manager-wait-if-another-instance-of-apt-is-running
 wait_for_apt() {
   while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do
@@ -38,13 +41,15 @@ apt-get -y autoremove
 pip3 install --upgrade pip
 hash -r pip
 
-mkdir /tmp/aws-install
-pushd /tmp/aws-install
-curl -sL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-unzip awscliv2.zip
-./aws/install
-popd
-rm -rf /tmp/aws-install
+if [ "$INSTALL_TYPE" != 'ci' ]; then
+  mkdir /tmp/aws-install
+  pushd /tmp/aws-install
+  curl -sL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+  unzip awscliv2.zip
+  ./aws/install
+  popd
+  rm -rf /tmp/aws-install
+fi
 
 mkdir -p /root/.aws /home/ubuntu/.aws
 echo -e "[default]\nregion=us-east-1" | tee /root/.aws/config /home/ubuntu/.aws/config
@@ -105,7 +110,11 @@ LOKI_PASSWORD=$(get_conf /compiler-explorer/lokiPassword)
 
 mkdir -p /etc/grafana
 cp /infra/grafana/agent.yaml /etc/grafana/agent.yaml.tpl
-cp "/infra/grafana/make-config${MAKE_CONFIG_SUFFIX:-}.sh" /etc/grafana/make-config.sh
+if [ "${INSTALL_TYPE}" = "ci" ]; then
+  cp /infra/grafana/make-config-ci.sh /etc/grafana/make-config.sh
+else
+  cp /infra/grafana/make-config.sh /etc/grafana/make-config.sh
+fi
 cp /infra/grafana/grafana-agent.service /lib/systemd/system/grafana-agent.service
 systemctl daemon-reload
 systemctl enable grafana-agent
@@ -135,7 +144,11 @@ cat >/etc/ssmtp/revaliases <<EOF
 ubuntu:admin@compiler-explorer.com:email-smtp.us-east-1.amazonaws.com
 EOF
 
-chfn -f 'Compiler Explorer Admin' ubuntu
+if [ "${INSTALL_TYPE}" = "ci" ]; then
+  chfn -f 'Compiler Explorer Admin' ubuntu
+else
+  chfn -f 'Compiler Explorer Build Agent' ubuntu
+fi
 chmod 640 /etc/ssmtp/*
 
 mount -a

From 0ef4ccbf6338824771446b35c22bc704fa642c21 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 25 Nov 2022 11:01:28 -0600
Subject: [PATCH 120/161] Erk, that's embarrassing

---
 setup-common.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup-common.sh b/setup-common.sh
index db3c16e58..0bd62eb5e 100755
--- a/setup-common.sh
+++ b/setup-common.sh
@@ -4,7 +4,6 @@ set -exuo pipefail
 
 INSTALL_TYPE=${1:-non-ci}
 
-cock
 # https://askubuntu.com/questions/132059/how-to-make-a-package-manager-wait-if-another-instance-of-apt-is-running
 wait_for_apt() {
   while fuser /var/lib/dpkg/lock >/dev/null 2>&1; do

From 339f2fada3aa09e275464e5be739c713651e1af3 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 25 Nov 2022 12:08:12 -0600
Subject: [PATCH 121/161] More tweaks to ci setup

---
 setup-ci.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/setup-ci.sh b/setup-ci.sh
index 3dfb3249c..a1feb4fc3 100755
--- a/setup-ci.sh
+++ b/setup-ci.sh
@@ -8,3 +8,7 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "${DIR}"
 
 env EXTRA_NFS_ARGS="" "${DIR}/setup-common.sh" ci
+
+ln -s /efs/squash-images /opt/squash-images
+ln -s /efs/compiler-explorer /opt/compiler-explorer
+ln -s /efs/wine-stable /opt/wine-stable

From 47264f6f3e3bbc26f9fa181f00305ad24a349a34 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Sat, 26 Nov 2022 13:00:55 +0100
Subject: [PATCH 122/161] Fix rust installer

---
 bin/lib/installable/rust.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/lib/installable/rust.py b/bin/lib/installable/rust.py
index 3cf2564e8..0c6dea87a 100644
--- a/bin/lib/installable/rust.py
+++ b/bin/lib/installable/rust.py
@@ -46,7 +46,7 @@ def do_rust_install(self, staging: StagingDir, component: str, install_to: Path)
         untar_to = staging.path / "__temp_install__"
         self.install_context.fetch_url_and_pipe_to(staging, url, ["tar", "zxf", "-", "--strip-components=1"], untar_to)
         self.install_context.stage_command(
-            staging, ["./install.sh", f"--prefix={install_to}", "--verbose", "--without=rust-docs"], cwd=untar_to
+            staging, ["./install.sh", f"--prefix={install_to}", "--verbose"], cwd=untar_to
         )
         self.install_context.remove_dir(untar_to)
 

From 9a7240e0305de0456a9aa20a29544aa7d4ca033a Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sat, 26 Nov 2022 14:22:25 -0600
Subject: [PATCH 123/161] Fix make clean. Closes #861. Good spot @partouf

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index ca05292f7..c2f435896 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,7 @@ packer-builder: config.json  ## Builds the base image for the CE builder
 
 .PHONY: clean
 clean:  ## Cleans up anything
-	rm -rf $(POETRY)
+	rm -rf $(POETRY_HOME)
 
 .PHONY: update-admin
 update-admin:  ## Updates the admin website

From 77aabe1a181d71bc14fa5891de286fc6cbd69fda Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sat, 26 Nov 2022 14:34:59 -0600
Subject: [PATCH 124/161] Move to new syntax for output variables. See #849
 (#888)

---
 .github/workflows/bespoke-build.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/bespoke-build.yaml b/.github/workflows/bespoke-build.yaml
index e278165e0..18789c9c9 100644
--- a/.github/workflows/bespoke-build.yaml
+++ b/.github/workflows/bespoke-build.yaml
@@ -41,15 +41,15 @@ jobs:
             /dist | tee output.log
           echo ::endgroup::
           echo ::group::Translate docker output into GH variables
-          gawk -e 'match($0, /^ce-build-(\w+):(.*)/, m) { print "::set-output name=" m[1] "::" m[2]; }' output.log
-            echo ::endgroup::
+          gawk -e 'match($0, /^ce-build-(\w+):(.*)/, m) { print m[1] "=" m[2]; }' output.log >> "${GITHUB_OUTPUT}"
+          echo ::endgroup::
       - name: Get output filename and s3 dest
         shell: bash
         if: ${{ steps.build.outputs.status == 'OK' }}
         id: filenames
         run: |
-          echo ::set-output name=source::$(pwd)/dist/$(basename ${{ steps.build.outputs.output }})
-          echo ::set-output name=s3dest::s3://compiler-explorer/opt/$(basename ${{ steps.build.outputs.output }})
+          echo source=$(pwd)/dist/$(basename ${{ steps.build.outputs.output }}) >> "${GITHUB_OUTPUT}"
+          echo s3dest=s3://compiler-explorer/opt/$(basename ${{ steps.build.outputs.output }}) >> "${GITHUB_OUTPUT}"
       - name: Copy output to S3
         if: ${{ steps.build.outputs.status == 'OK' }}
         uses: prewk/s3-cp-action@v2

From c5cf1f5e909e0d6a121563a58a230acc4a550fc0 Mon Sep 17 00:00:00 2001
From: kasperk81 <83082615+kasperk81@users.noreply.github.com>
Date: Sun, 27 Nov 2022 01:29:58 +0200
Subject: [PATCH 125/161] add trunk in dotnet (#887)

* add trunk in dotnet
---
 bin/yaml/dotnet.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index 59c7c8388..662224ed9 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -5,7 +5,10 @@ compilers:
     untar_dir: Core_Root
     check_exe: .dotnet/dotnet --version
     targets:
+      - name: trunk
+        if: nightly
+        always_install: true
+      - v7.0.0
       - name: v6.0.11
         check_env:
           - DOTNET_CLI_HOME=/tmp
-      - v7.0.0

From 22a57c645f9af8844b67647a7c2c4420d6cbe973 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Sun, 27 Nov 2022 00:41:31 +0100
Subject: [PATCH 126/161] add our own branch

---
 bin/yaml/cpp.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index eaa62dbed..6f42853b7 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -839,6 +839,7 @@ compilers:
           - p1061-trunk
           - reflection-trunk
           - rocm-trunk
+          - ce-trunk
           - name: llvmflang-trunk
             check_exe: bin/flang
             after_stage_script:

From ae0b630e58bfda8dcb79ef619d2a181904e33f80 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Poulhi=C3=A8s?= <dkm@kataplop.net>
Date: Mon, 28 Nov 2022 18:44:49 +0100
Subject: [PATCH 127/161] Add Zip 0.10.0 (#889)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

https://ziglang.org/download/#release-0.10.0

refs https://github.com/compiler-explorer/compiler-explorer/issues/4357

Signed-off-by: Marc Poulhiès <dkm@kataplop.net>
---
 bin/yaml/zig.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/yaml/zig.yaml b/bin/yaml/zig.yaml
index 61d56b6aa..040a4253f 100644
--- a/bin/yaml/zig.yaml
+++ b/bin/yaml/zig.yaml
@@ -18,6 +18,7 @@ compilers:
       - 0.7.1
       - 0.8.0
       - 0.9.0
+      - 0.10.0
     nightly:
       if: nightly
       type: restQueryTarballs

From 01598e2e9e919a79f865a99f132f42a52c3fb41c Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Mon, 28 Nov 2022 23:44:52 +0100
Subject: [PATCH 128/161] fix #880 (#890)

---
 init/start.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init/start.sh b/init/start.sh
index 91c215dfe..37e42f489 100755
--- a/init/start.sh
+++ b/init/start.sh
@@ -89,7 +89,7 @@ install_asmparser() {
 
 install_ninja() {
     rm -f /usr/local/bin/ninja
-    cp /opt/compiler-explorer/ninja/ninja /usr/local/bin
+    cp "$(readlink -f /opt/compiler-explorer/ninja/ninja)" /usr/local/bin
 }
 
 LOG_DEST_HOST=$(get_conf /compiler-explorer/logDestHost)

From 1d06cfcb279ccaaec37aeb796035cdfaff24bd12 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Tue, 29 Nov 2022 00:02:00 +0100
Subject: [PATCH 129/161] new staging ami

---
 terraform/lc.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/lc.tf b/terraform/lc.tf
index 96f755d3f..b188d1c89 100644
--- a/terraform/lc.tf
+++ b/terraform/lc.tf
@@ -1,6 +1,6 @@
 locals {
   image_id          = "ami-08b5bae0de035991b"
-  staging_image_id  = "ami-08b5bae0de035991b"
+  staging_image_id  = "ami-01f471a79a5b6ebb6"
   beta_image_id     = "ami-08b5bae0de035991b"
   gpu_image_id      = "ami-05b58707118bd281d"
   staging_user_data = base64encode("staging")

From 878fad87df1a04031a53bf17efdc5a3c57e832f4 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Tue, 29 Nov 2022 00:39:31 +0100
Subject: [PATCH 130/161] bugfix staging

---
 site-staging.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/site-staging.sh b/site-staging.sh
index 3aa86efa7..63b1a1c25 100644
--- a/site-staging.sh
+++ b/site-staging.sh
@@ -2,3 +2,4 @@
 # Sourced when running in staging mode
 export BRANCH=staging
 export NODE_ENV=production
+export CE_PROP_ENV="${CE_PROP_ENV} --env staging"

From e57267f01b9bc45a9940de58ee4730db5efa187c Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Tue, 29 Nov 2022 00:42:05 +0100
Subject: [PATCH 131/161] fix beta

---
 site-beta.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/site-beta.sh b/site-beta.sh
index f1dd9941e..bb57bc4ad 100644
--- a/site-beta.sh
+++ b/site-beta.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
 # Sourced when running in beta mode
 export BRANCH=beta
-export EXTRA_ARGS='--env beta --ensureNoIdClash'
+export CE_PROP_ENV="${CE_PROP_ENV} --env beta"
 export NODE_ENV=production

From 25e765044551762df2d8cbe45a568e6f0e0a1171 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Tue, 29 Nov 2022 00:57:16 +0100
Subject: [PATCH 132/161] new amis

---
 terraform/lc.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform/lc.tf b/terraform/lc.tf
index b188d1c89..c5e380eb3 100644
--- a/terraform/lc.tf
+++ b/terraform/lc.tf
@@ -1,7 +1,7 @@
 locals {
-  image_id          = "ami-08b5bae0de035991b"
-  staging_image_id  = "ami-01f471a79a5b6ebb6"
-  beta_image_id     = "ami-08b5bae0de035991b"
+  image_id          = "ami-030d64ab670a5148c"
+  staging_image_id  = "ami-030d64ab670a5148c"
+  beta_image_id     = "ami-030d64ab670a5148c"
   gpu_image_id      = "ami-05b58707118bd281d"
   staging_user_data = base64encode("staging")
   beta_user_data    = base64encode("beta")

From e89c75c2bf854b3501c1da2281dfac152466837f Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Wed, 30 Nov 2022 02:39:16 +0100
Subject: [PATCH 133/161] Update dotnet.yaml

---
 bin/yaml/dotnet.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index 662224ed9..e38ce2578 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -8,6 +8,7 @@ compilers:
       - name: trunk
         if: nightly
         always_install: true
+        type: nightly
       - v7.0.0
       - name: v6.0.11
         check_env:

From f72082bcda813d0d26fc8bc23790b413eda68cd8 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Wed, 30 Nov 2022 02:54:33 +0100
Subject: [PATCH 134/161] Update dotnet.yaml

---
 bin/yaml/dotnet.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index e38ce2578..0f5f759c5 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -7,7 +7,7 @@ compilers:
     targets:
       - name: trunk
         if: nightly
-        always_install: true
+        untar_dir: Core_Root
         type: nightly
       - v7.0.0
       - name: v6.0.11

From 4965282051909d5d2742015e19bcd70c009635a0 Mon Sep 17 00:00:00 2001
From: Patrick Quist <partouf@gmail.com>
Date: Wed, 30 Nov 2022 03:00:40 +0100
Subject: [PATCH 135/161] Update dotnet.yaml

---
 bin/yaml/dotnet.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bin/yaml/dotnet.yaml b/bin/yaml/dotnet.yaml
index 0f5f759c5..b5cbd0724 100644
--- a/bin/yaml/dotnet.yaml
+++ b/bin/yaml/dotnet.yaml
@@ -7,7 +7,6 @@ compilers:
     targets:
       - name: trunk
         if: nightly
-        untar_dir: Core_Root
         type: nightly
       - v7.0.0
       - name: v6.0.11

From 1cfb80d2d404416e831249ed1f7fb1d25f0e35ff Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Sat, 3 Dec 2022 16:35:50 +0100
Subject: [PATCH 136/161] fix #891

---
 bin/yaml/libraries.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index 8b4685821..b212734d0 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -1000,9 +1000,10 @@ libraries:
         build_type: none
         check_file: README.md
         method: nightlyclone
-        path_name: libs/folly
         repo: facebook/folly
         type: github
+        targets:
+        - trunk
       glm:
         build_type: none
         check_file: readme.md

From 573b9b9b1ca41a0a1ca4bb0dde0a87f48304d4f9 Mon Sep 17 00:00:00 2001
From: Partouf <partouf@gmail.com>
Date: Wed, 7 Dec 2022 00:30:54 +0100
Subject: [PATCH 137/161] add some libs

---
 bin/yaml/libraries.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index b212734d0..218b8f5a5 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -2013,6 +2013,11 @@ libraries:
       targets:
       - 0.8.8
       type: cratesio
+    color-eyre:
+      build_type: cargo
+      targets:
+      - 0.6.2
+      type: cratesio
     digest:
       build_type: cargo
       targets:
@@ -2028,6 +2033,11 @@ libraries:
       targets:
       - 0.9.0
       type: cratesio
+    eyre:
+      build_type: cargo
+      targets:
+      - 0.6.8
+      type: cratesio
     fnv:
       build_type: cargo
       targets:

From 82ecd81400b459ac53c69d51e9afe737a66a0af3 Mon Sep 17 00:00:00 2001
From: Michael Debertol <michael.debertol@gmail.com>
Date: Sun, 11 Dec 2022 00:05:32 +0100
Subject: [PATCH 138/161] Add dart 2.18 and dart dev version (#895)

---
 bin/yaml/dart.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/bin/yaml/dart.yaml b/bin/yaml/dart.yaml
index d086fcd59..1d981aebb 100644
--- a/bin/yaml/dart.yaml
+++ b/bin/yaml/dart.yaml
@@ -9,6 +9,7 @@ compilers:
       - find -executable -print0 | xargs -0 chmod og+x
     url: https://storage.googleapis.com/dart-archive/channels/stable/release/{name}/sdk/dartsdk-linux-x64-release.zip
     targets:
+      - 2.18.5
       - 2.17.1
       - 2.16.1
       - 2.15.1
@@ -19,3 +20,9 @@ compilers:
       - 2.10.5
       - 2.9.3
       - 2.8.4
+    nightly:
+      if: nightly
+      install_always: true
+      url: https://storage.googleapis.com/dart-archive/channels/{name}/release/latest/sdk/dartsdk-linux-x64-release.zip
+      targets:
+        - dev

From 55f7fd6043113e53f3037331c29296ad43674e6a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 11 Dec 2022 00:25:20 +0100
Subject: [PATCH 139/161] Bump certifi from 2022.9.24 to 2022.12.7 (#893)

Bumps [certifi](https://github.com/certifi/python-certifi) from 2022.9.24 to 2022.12.7.
- [Release notes](https://github.com/certifi/python-certifi/releases)
- [Commits](https://github.com/certifi/python-certifi/compare/2022.09.24...2022.12.07)

---
updated-dependencies:
- dependency-name: certifi
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 poetry.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 7eed1c908..1fcf2d660 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -141,7 +141,7 @@ redis = ["redis (>=2.10.5)"]
 
 [[package]]
 name = "certifi"
-version = "2022.9.24"
+version = "2022.12.7"
 description = "Python package for providing Mozilla's CA Bundle."
 category = "main"
 optional = false
@@ -856,8 +856,8 @@ cachecontrol = [
     {file = "CacheControl-0.12.11.tar.gz", hash = "sha256:a5b9fcc986b184db101aa280b42ecdcdfc524892596f606858e0b7a8b4d9e144"},
 ]
 certifi = [
-    {file = "certifi-2022.9.24-py3-none-any.whl", hash = "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382"},
-    {file = "certifi-2022.9.24.tar.gz", hash = "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14"},
+    {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"},
+    {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"},
 ]
 cffi = [
     {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"},

From ff22d76d1b9e9f274685d6743eea5b10fda27f91 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 12 Dec 2022 20:06:23 -0600
Subject: [PATCH 140/161] Alternative poetry setup (#897)

The virtualenv thing was dodgy and it turns out that we can actually directly create the .venv directory as part of peotry itself, with the correct configuration. So, do that!

If you get a conflict, remove the autogenerated poetry.toml and replace it with the one from source control.
---
 .gitignore  |  2 +-
 Makefile    | 10 ++++------
 poetry.toml |  2 ++
 3 files changed, 7 insertions(+), 7 deletions(-)
 create mode 100644 poetry.toml

diff --git a/.gitignore b/.gitignore
index dd52ac3f9..4ed560754 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,4 +11,4 @@ node_modules
 /.mypy_cache
 /.dist
 /.poetry
-/poetry.toml
+/.venv
diff --git a/Makefile b/Makefile
index c2f435896..cceec2989 100644
--- a/Makefile
+++ b/Makefile
@@ -2,8 +2,8 @@
 
 export POETRY_HOME=$(CURDIR)/.poetry
 POETRY:=$(POETRY_HOME)/bin/poetry
-POETRY_VENVS=$(POETRY_HOME)/virtualenvs
-POETRY_DEPS:=$(POETRY_VENVS)/.deps
+POETRY_VENV=$(CURDIR)/.venv
+POETRY_DEPS:=$(POETRY_VENV)/.deps
 SYS_PYTHON:=$(shell env PATH='/bin:/usr/bin:/usr/local/bin:$(PATH)' bash -c "command -v python3.11 || command -v python3.10 || command -v python3.9 || echo .python-not-found")
 export PYTHONPATH=$(CURDIR)/bin
 
@@ -49,8 +49,8 @@ packer-builder: config.json  ## Builds the base image for the CE builder
 	$(PACKER) build -timestamp-ui -var-file=config.json $(EXTRA_ARGS) packer-builder.json
 
 .PHONY: clean
-clean:  ## Cleans up anything
-	rm -rf $(POETRY_HOME)
+clean:  ## Cleans up everything
+	rm -rf $(POETRY_HOME) $(POETRY_VENV)
 
 .PHONY: update-admin
 update-admin:  ## Updates the admin website
@@ -61,8 +61,6 @@ ce: $(POETRY) $(POETRY_DEPS)  ## Installs and configures the python environment
 $(POETRY): $(SYS_PYTHON) poetry.toml
 	curl -sSL https://install.python-poetry.org | $(SYS_PYTHON) -
 	@touch $@
-poetry.toml: Makefile
-	echo "# automatically generated by Makefile\nvirtualenvs.path = \"$(POETRY_VENVS)\"" > poetry.toml
 
 $(POETRY_DEPS): $(POETRY) pyproject.toml poetry.lock
 	$(POETRY) install --sync
diff --git a/poetry.toml b/poetry.toml
new file mode 100644
index 000000000..ab1033bd3
--- /dev/null
+++ b/poetry.toml
@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

From 25115be67167e743254c424f35b4453cd82c897a Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Wed, 14 Dec 2022 18:47:15 +0100
Subject: [PATCH 141/161] test runner wrapperscripts

---
 .github/workflows/compiler-discovery.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/compiler-discovery.yml b/.github/workflows/compiler-discovery.yml
index 57d5e6c80..9d1fc8991 100644
--- a/.github/workflows/compiler-discovery.yml
+++ b/.github/workflows/compiler-discovery.yml
@@ -28,15 +28,22 @@ jobs:
     steps:
     - name: Update infra
       run: cd /home/ubuntu/infra && git pull
+      shell: bash
     - name: Set build number ${{ github.event.inputs.buildnumber }}
       run: ce --env runner builds set_current ${{ github.event.inputs.buildnumber }} --branch ${{ github.event.inputs.branch }} --confirm
+      shell: bash
     - name: Start runner
       run: ce runner start
+      shell: bash
     - name: Update infra on runner
       run: ce runner pull
+      shell: bash
     - name: Do Compiler discovery
       run: ce runner discovery
+      shell: bash
     - name: Upload discovered compilers
       run: ce runner uploaddiscovery ${{ github.event.inputs.environment }} ${{ github.event.inputs.buildnumber }}
+      shell: bash
     - name: Shut down runner
       run: ce runner stop
+      shell: bash

From d9139681d4f886e5d8a2ab0bd90c18eac2220fb9 Mon Sep 17 00:00:00 2001
From: partouf <partouf@gmail.com>
Date: Wed, 14 Dec 2022 19:24:56 +0100
Subject: [PATCH 142/161] fix ce scripts exitcode

---
 bin/lib/ce.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bin/lib/ce.py b/bin/lib/ce.py
index 84863f760..d350f0431 100755
--- a/bin/lib/ce.py
+++ b/bin/lib/ce.py
@@ -11,10 +11,13 @@
 def main():
     try:
         cli(prog_name="ce")  # pylint: disable=unexpected-keyword-arg,no-value-for-parameter
-    except (KeyboardInterrupt, SystemExit):
+    except KeyboardInterrupt:
         # print empty line so terminal prompt doesn't end up on the end of some
         # of our own program output
         print()
+    except SystemExit:
+        print()
+        raise
 
 
 if __name__ == "__main__":

From 9e918dfa41b1b2011aa6972986a04b7c55cd2815 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Thu, 15 Dec 2022 07:50:41 -0600
Subject: [PATCH 143/161] Add ISPC builder

---
 .github/workflows/bespoke-build.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/bespoke-build.yaml b/.github/workflows/bespoke-build.yaml
index 18789c9c9..f6e4420f1 100644
--- a/.github/workflows/bespoke-build.yaml
+++ b/.github/workflows/bespoke-build.yaml
@@ -15,6 +15,7 @@ on:
         - misc
         - ocaml
         - python
+        - ispc
       version:
         description: The version to build (or architecture and version for cross)
         required: true

From a0476cb08caeae117274ffca3187980ada5624ef Mon Sep 17 00:00:00 2001
From: Miguel Ojeda <ojeda@users.noreply.github.com>
Date: Thu, 15 Dec 2022 18:11:58 +0100
Subject: [PATCH 144/161] Rust 1.66.0 (#902)

Signed-off-by: Miguel Ojeda <ojeda@kernel.org>

Signed-off-by: Miguel Ojeda <ojeda@kernel.org>
---
 bin/yaml/rust.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/yaml/rust.yaml b/bin/yaml/rust.yaml
index ddf2d2e6c..d53afe7c9 100644
--- a/bin/yaml/rust.yaml
+++ b/bin/yaml/rust.yaml
@@ -78,6 +78,7 @@ compilers:
         - 1.63.0
         - 1.64.0
         - 1.65.0
+        - 1.66.0
       nightly:
         if: nightly
         targets:

From ce0f0276f58ccc095e5c07d0b3dfd96fe33d0780 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marc=20Poulhi=C3=A8s?= <dkm@kataplop.net>
Date: Thu, 15 Dec 2022 21:40:52 +0100
Subject: [PATCH 145/161] add: gcc bpf cross compiler (#894)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add GCC for C and C++, for BPF target.

refs https://github.com/compiler-explorer/compiler-explorer/issues/4408

Signed-off-by: Marc Poulhiès <dkm@kataplop.net>
---
 bin/yaml/cpp.yaml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/bin/yaml/cpp.yaml b/bin/yaml/cpp.yaml
index 6f42853b7..7b60d37df 100644
--- a/bin/yaml/cpp.yaml
+++ b/bin/yaml/cpp.yaml
@@ -278,6 +278,17 @@ compilers:
             path_name_prefix: gcc-trunk
             targets:
               - trunk
+        bpf:
+          arch_prefix: "{subdir}-unknown"
+          check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"
+          subdir: bpf
+          nightly:
+            if: nightly
+            type: nightly
+            compiler_name: bpf-gcc-trunk
+            path_name_prefix: gcc-trunk
+            targets:
+              - trunk
         mips:
           - arch_prefix: "{subdir}-unknown-linux-gnu"
             check_exe: "{arch_prefix}/bin/{arch_prefix}-g++ --version"

From 8791049fa6ea6285feac7c08f2aac9be68143f9c Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Thu, 15 Dec 2022 22:15:45 -0600
Subject: [PATCH 146/161] Install our own ispcs

---
 bin/yaml/ispc.yaml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/bin/yaml/ispc.yaml b/bin/yaml/ispc.yaml
index 7551701a4..fa867aea2 100644
--- a/bin/yaml/ispc.yaml
+++ b/bin/yaml/ispc.yaml
@@ -30,9 +30,8 @@ compilers:
           - 1.9.1
     nightly:
       if: nightly
-      type: nightlytarballs
-      url: "https://ci.appveyor.com/api/projects/ispc/ispc/artifacts/build/ispc-trunk-linux.tar.gz?job=Environment: APPVEYOR_BUILD_WORKER_IMAGE=Ubuntu1804, LLVM_VERSION=latest"
-      dir: ispc-{name}
-      untar_dir: ispc-{name}-linux
+      compiler_name: ispc-{name}
+      type: nightly
       targets:
         - trunk
+        - templates_new-trunk

From 0596cb12956e245acf8438341022c06503293578 Mon Sep 17 00:00:00 2001
From: Madhur Chauhan <madhur4127@gmail.com>
Date: Fri, 16 Dec 2022 18:48:23 +0530
Subject: [PATCH 147/161] update pahole and elfutils (#899)

* update pahole and elfutils nightly
---
 update_compilers/install_binaries.sh | 75 +++++++++++++---------------
 1 file changed, 34 insertions(+), 41 deletions(-)

diff --git a/update_compilers/install_binaries.sh b/update_compilers/install_binaries.sh
index ca6e67c95..ba135b917 100755
--- a/update_compilers/install_binaries.sh
+++ b/update_compilers/install_binaries.sh
@@ -8,47 +8,40 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 #########################
 # pahole
-
-TARGET_PAHOLE_VERSION=v1.19
-CURRENT_PAHOLE_VERSION=""
-if [[ -f "${OPT}/pahole/bin/pahole" ]]; then
-    CURRENT_PAHOLE_VERSION=$("${OPT}/pahole/bin/pahole" --version)
-fi
-
-if [[ "$TARGET_PAHOLE_VERSION" != "$CURRENT_PAHOLE_VERSION" ]]; then
-    rm -Rf "${OPT}/pahole"
-    mkdir -p "${OPT}/pahole"
-
-    mkdir -p /tmp/build
-    pushd /tmp/build
-
-    # Install elfutils for libelf and libdwarf
-    fetch https://sourceware.org/elfutils/ftp/0.182/elfutils-0.182.tar.bz2 | tar jxf -
-    pushd elfutils-0.182
-    ./configure --prefix=/opt/compiler-explorer/pahole --program-prefix="eu-" --enable-deterministic-archives --disable-debuginfod --disable-libdebuginfod
-    make "-j$(nproc)"
-    make install
-    popd
-
-    rm -Rf /tmp/build/pahole
-
-    git clone -q https://git.kernel.org/pub/scm/devel/pahole/pahole.git pahole
-    git -C pahole checkout v1.19
-    git -C pahole submodule sync
-    git -C pahole submodule update --init
-    pushd pahole
-    "${OPT}/cmake/bin/cmake" \
-        -D CMAKE_INSTALL_PREFIX:PATH="${OPT}/pahole" \
-        -D__LIB=lib \
-        -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \
-        .
-    make "-j$(nproc)"
-    make install
-    popd
-
-    popd
-    rm -rf /tmp/build
-fi
+PAHOLE_INSTALL_DIR=$OPT/pahole
+
+## install elfutils for libelf and libdwarf
+ELFUTILS_VERSION=0.188
+rm -Rf /tmp/elfutils-$ELFUTILS_VERSION
+fetch https://sourceware.org/elfutils/ftp/$ELFUTILS_VERSION/elfutils-$ELFUTILS_VERSION.tar.bz2 | tar jxf - -C /tmp/
+pushd /tmp/elfutils-$ELFUTILS_VERSION
+./configure --prefix=$PAHOLE_INSTALL_DIR --program-prefix="eu-" --enable-deterministic-archives --disable-debuginfod --disable-libdebuginfod
+make "-j$(nproc)"
+make install
+popd
+rm -Rf /tmp/elfutils-$ELFUTILS_VERSION
+
+## cleanup before build
+PAHOLE_BUILD_DIR=/tmp/build/pahole
+rm -rf $PAHOLE_BUILD_DIR
+mkdir -p $PAHOLE_BUILD_DIR
+pushd $PAHOLE_BUILD_DIR
+
+## download and install
+git clone --branch master --single-branch -q https://git.kernel.org/pub/scm/devel/pahole/pahole.git pahole
+pushd pahole
+git submodule sync
+git submodule update --init
+"${OPT}/cmake/bin/cmake" \
+    -D CMAKE_INSTALL_PREFIX:PATH="$PAHOLE_INSTALL_DIR" \
+    -D__LIB=lib \
+    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \
+    .
+make "-j$(nproc)"
+make install
+popd
+popd
+rm -rf $PAHOLE_BUILD_DIR
 
 #########################
 # x86-to-6502 old version

From 318d1aaa12a577a01852011f8befa01b20ad8f2c Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 16 Dec 2022 07:20:52 -0600
Subject: [PATCH 148/161] Fix tests

---
 update_compilers/install_binaries.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/update_compilers/install_binaries.sh b/update_compilers/install_binaries.sh
index ba135b917..2c7ea2b96 100755
--- a/update_compilers/install_binaries.sh
+++ b/update_compilers/install_binaries.sh
@@ -15,7 +15,7 @@ ELFUTILS_VERSION=0.188
 rm -Rf /tmp/elfutils-$ELFUTILS_VERSION
 fetch https://sourceware.org/elfutils/ftp/$ELFUTILS_VERSION/elfutils-$ELFUTILS_VERSION.tar.bz2 | tar jxf - -C /tmp/
 pushd /tmp/elfutils-$ELFUTILS_VERSION
-./configure --prefix=$PAHOLE_INSTALL_DIR --program-prefix="eu-" --enable-deterministic-archives --disable-debuginfod --disable-libdebuginfod
+./configure --prefix="$PAHOLE_INSTALL_DIR" --program-prefix="eu-" --enable-deterministic-archives --disable-debuginfod --disable-libdebuginfod
 make "-j$(nproc)"
 make install
 popd

From 8c0432ba425a2e6017c8fce4d47d56622514fce9 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 16 Dec 2022 08:13:08 -0600
Subject: [PATCH 149/161] Revert "update pahole and elfutils (#899)"

This reverts commit 8b74dfc0dd015bddbc18ca2dc338c3168800482c.
---
 update_compilers/install_binaries.sh | 75 +++++++++++++++-------------
 1 file changed, 41 insertions(+), 34 deletions(-)

diff --git a/update_compilers/install_binaries.sh b/update_compilers/install_binaries.sh
index 2c7ea2b96..ca6e67c95 100755
--- a/update_compilers/install_binaries.sh
+++ b/update_compilers/install_binaries.sh
@@ -8,40 +8,47 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 #########################
 # pahole
-PAHOLE_INSTALL_DIR=$OPT/pahole
-
-## install elfutils for libelf and libdwarf
-ELFUTILS_VERSION=0.188
-rm -Rf /tmp/elfutils-$ELFUTILS_VERSION
-fetch https://sourceware.org/elfutils/ftp/$ELFUTILS_VERSION/elfutils-$ELFUTILS_VERSION.tar.bz2 | tar jxf - -C /tmp/
-pushd /tmp/elfutils-$ELFUTILS_VERSION
-./configure --prefix="$PAHOLE_INSTALL_DIR" --program-prefix="eu-" --enable-deterministic-archives --disable-debuginfod --disable-libdebuginfod
-make "-j$(nproc)"
-make install
-popd
-rm -Rf /tmp/elfutils-$ELFUTILS_VERSION
-
-## cleanup before build
-PAHOLE_BUILD_DIR=/tmp/build/pahole
-rm -rf $PAHOLE_BUILD_DIR
-mkdir -p $PAHOLE_BUILD_DIR
-pushd $PAHOLE_BUILD_DIR
-
-## download and install
-git clone --branch master --single-branch -q https://git.kernel.org/pub/scm/devel/pahole/pahole.git pahole
-pushd pahole
-git submodule sync
-git submodule update --init
-"${OPT}/cmake/bin/cmake" \
-    -D CMAKE_INSTALL_PREFIX:PATH="$PAHOLE_INSTALL_DIR" \
-    -D__LIB=lib \
-    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \
-    .
-make "-j$(nproc)"
-make install
-popd
-popd
-rm -rf $PAHOLE_BUILD_DIR
+
+TARGET_PAHOLE_VERSION=v1.19
+CURRENT_PAHOLE_VERSION=""
+if [[ -f "${OPT}/pahole/bin/pahole" ]]; then
+    CURRENT_PAHOLE_VERSION=$("${OPT}/pahole/bin/pahole" --version)
+fi
+
+if [[ "$TARGET_PAHOLE_VERSION" != "$CURRENT_PAHOLE_VERSION" ]]; then
+    rm -Rf "${OPT}/pahole"
+    mkdir -p "${OPT}/pahole"
+
+    mkdir -p /tmp/build
+    pushd /tmp/build
+
+    # Install elfutils for libelf and libdwarf
+    fetch https://sourceware.org/elfutils/ftp/0.182/elfutils-0.182.tar.bz2 | tar jxf -
+    pushd elfutils-0.182
+    ./configure --prefix=/opt/compiler-explorer/pahole --program-prefix="eu-" --enable-deterministic-archives --disable-debuginfod --disable-libdebuginfod
+    make "-j$(nproc)"
+    make install
+    popd
+
+    rm -Rf /tmp/build/pahole
+
+    git clone -q https://git.kernel.org/pub/scm/devel/pahole/pahole.git pahole
+    git -C pahole checkout v1.19
+    git -C pahole submodule sync
+    git -C pahole submodule update --init
+    pushd pahole
+    "${OPT}/cmake/bin/cmake" \
+        -D CMAKE_INSTALL_PREFIX:PATH="${OPT}/pahole" \
+        -D__LIB=lib \
+        -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \
+        .
+    make "-j$(nproc)"
+    make install
+    popd
+
+    popd
+    rm -rf /tmp/build
+fi
 
 #########################
 # x86-to-6502 old version

From 28791460e653fef3934ab7fb0a918bf20d86fdac Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 16 Dec 2022 08:19:42 -0600
Subject: [PATCH 150/161] Build in release

---
 update_compilers/install_binaries.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/update_compilers/install_binaries.sh b/update_compilers/install_binaries.sh
index ca6e67c95..6db95b680 100755
--- a/update_compilers/install_binaries.sh
+++ b/update_compilers/install_binaries.sh
@@ -38,6 +38,7 @@ if [[ "$TARGET_PAHOLE_VERSION" != "$CURRENT_PAHOLE_VERSION" ]]; then
     git -C pahole submodule update --init
     pushd pahole
     "${OPT}/cmake/bin/cmake" \
+        -DCMAKE_BUILD_TYPE=Release \
         -D CMAKE_INSTALL_PREFIX:PATH="${OPT}/pahole" \
         -D__LIB=lib \
         -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \

From 688da99a35c136e1000cdc6928e2178f22aedd21 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Fri, 16 Dec 2022 09:51:26 -0600
Subject: [PATCH 151/161] pahole from nightly ce_install

---
 bin/yaml/tools.yaml                  |  6 ++++
 update_compilers/install_binaries.sh | 45 ----------------------------
 2 files changed, 6 insertions(+), 45 deletions(-)

diff --git a/bin/yaml/tools.yaml b/bin/yaml/tools.yaml
index 7fa90f9b7..3add58303 100644
--- a/bin/yaml/tools.yaml
+++ b/bin/yaml/tools.yaml
@@ -101,3 +101,9 @@ tools:
     check_exe: bin/patchelf --version
     targets:
       - 0.15.0
+  pahole:
+    if: nightly
+    type: nightly
+    check_exe: bin/pahole
+    targets:
+      - trunk
diff --git a/update_compilers/install_binaries.sh b/update_compilers/install_binaries.sh
index 6db95b680..008c212ad 100755
--- a/update_compilers/install_binaries.sh
+++ b/update_compilers/install_binaries.sh
@@ -6,51 +6,6 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=common.inc
 . "${DIR}/common.inc"
 
-#########################
-# pahole
-
-TARGET_PAHOLE_VERSION=v1.19
-CURRENT_PAHOLE_VERSION=""
-if [[ -f "${OPT}/pahole/bin/pahole" ]]; then
-    CURRENT_PAHOLE_VERSION=$("${OPT}/pahole/bin/pahole" --version)
-fi
-
-if [[ "$TARGET_PAHOLE_VERSION" != "$CURRENT_PAHOLE_VERSION" ]]; then
-    rm -Rf "${OPT}/pahole"
-    mkdir -p "${OPT}/pahole"
-
-    mkdir -p /tmp/build
-    pushd /tmp/build
-
-    # Install elfutils for libelf and libdwarf
-    fetch https://sourceware.org/elfutils/ftp/0.182/elfutils-0.182.tar.bz2 | tar jxf -
-    pushd elfutils-0.182
-    ./configure --prefix=/opt/compiler-explorer/pahole --program-prefix="eu-" --enable-deterministic-archives --disable-debuginfod --disable-libdebuginfod
-    make "-j$(nproc)"
-    make install
-    popd
-
-    rm -Rf /tmp/build/pahole
-
-    git clone -q https://git.kernel.org/pub/scm/devel/pahole/pahole.git pahole
-    git -C pahole checkout v1.19
-    git -C pahole submodule sync
-    git -C pahole submodule update --init
-    pushd pahole
-    "${OPT}/cmake/bin/cmake" \
-        -DCMAKE_BUILD_TYPE=Release \
-        -D CMAKE_INSTALL_PREFIX:PATH="${OPT}/pahole" \
-        -D__LIB=lib \
-        -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=TRUE \
-        .
-    make "-j$(nproc)"
-    make install
-    popd
-
-    popd
-    rm -rf /tmp/build
-fi
-
 #########################
 # x86-to-6502 old version
 

From 42232400e8db92c6aadb5653bcb3e53655ed13c1 Mon Sep 17 00:00:00 2001
From: Alfredo Correa <alfredo.correa+work@gmail.com>
Date: Sat, 17 Dec 2022 14:04:35 -0800
Subject: [PATCH 152/161] Add Josutti's Belleviews library (#900)

https://github.com/josuttis/belleviews
---
 bin/yaml/libraries.yaml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index 218b8f5a5..875b5c93b 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -759,6 +759,14 @@ libraries:
         targets:
         - trunk
         type: github
+      belleviews:
+        build_type: none
+        check_file: sources/belleviews.hpp
+        method: nightlyclone
+        repo: josuttis/belleviews
+        targets:
+        - trunk
+        type: github
       blaze:
         build_type: none
         check_file: LICENSE

From 13e71ab614a64ef34f092995200afa826b3aec78 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Sat, 17 Dec 2022 18:29:33 -0600
Subject: [PATCH 153/161] Alternative way of specifying instance types

---
 terraform/asg.tf | 46 ++++++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/terraform/asg.tf b/terraform/asg.tf
index b1fe2f91c..936dadd8f 100644
--- a/terraform/asg.tf
+++ b/terraform/asg.tf
@@ -3,12 +3,8 @@ locals {
   // As of Oct 3 2022, startups started taking >2m
   grace_period = 160
   cooldown     = 180
-  // Current c5 on-demand price is 0.085. Yearly pre-pay is 0.05 (so this is same as prepaying a year)
-  // Historically we pay ~ 0.035
-  spot_price        = "0.05"
 }
 
-
 resource "aws_autoscaling_group" "prod-mixed" {
   lifecycle {
     create_before_destroy = true
@@ -25,15 +21,13 @@ resource "aws_autoscaling_group" "prod-mixed" {
 
   mixed_instances_policy {
     instances_distribution {
-      on_demand_allocation_strategy            = "prioritized"
       // This base value is zero so we don't have any non-spot instances. We may wish to bump this if we have issues
       // getting spot capacity.
+      on_demand_allocation_strategy = "lowest-price"
       on_demand_base_capacity                  = 0
       on_demand_percentage_above_base_capacity = 0
-      spot_allocation_strategy                 = "lowest-price"
-      spot_instance_pools                      = 2
-      // we may consider upping this if we have issues fulfilling
-      spot_max_price                           = local.spot_price
+      spot_allocation_strategy                 = "price-capacity-optimized"
+      spot_instance_pools = 0
     }
     launch_template {
       launch_template_specification {
@@ -41,13 +35,21 @@ resource "aws_autoscaling_group" "prod-mixed" {
         version            = "$Latest"
       }
       override {
-        instance_type = "c6i.large"
-      }
-      override {
-        instance_type = "c5.large"
-      }
-      override {
-        instance_type = "c5a.large"
+        instance_requirements {
+          memory_mib {
+            min = 4096
+          }
+          vcpu_count {
+            min = 2
+          }
+          memory_gib_per_vcpu {
+            min = 2
+          }
+          accelerator_count {
+            max = 0
+          }
+          spot_max_price_percentage_over_lowest_price = 150
+        }
       }
     }
   }
@@ -137,10 +139,10 @@ resource "aws_autoscaling_group" "gpu" {
   default_cooldown          = local.cooldown
   health_check_grace_period = local.grace_period
   health_check_type         = "ELB"
-  max_size            = 2
-  min_size            = 0
-  name                = "gpu"
-  vpc_zone_identifier = local.subnets
+  max_size                  = 2
+  min_size                  = 0
+  name                      = "gpu"
+  vpc_zone_identifier       = local.subnets
 
   mixed_instances_policy {
     instances_distribution {
@@ -149,8 +151,8 @@ resource "aws_autoscaling_group" "gpu" {
       // getting spot capacity.
       on_demand_base_capacity                  = 0
       on_demand_percentage_above_base_capacity = 0
-      spot_allocation_strategy                 = "lowest-price"
-      spot_instance_pools                      = 2
+      spot_allocation_strategy                 = "price-capacity-optimized"
+      spot_instance_pools = 0
     }
     launch_template {
       launch_template_specification {

From 66f1ab1787271998815cc74df79db9179a053886 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 19 Dec 2022 19:57:34 -0600
Subject: [PATCH 154/161] Fix tests

---
 bin/lib/cefs/root.py  |  2 +-
 bin/test/cefs_test.py | 24 ++++++++++++------------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/bin/lib/cefs/root.py b/bin/lib/cefs/root.py
index 7f5f43626..adb754832 100644
--- a/bin/lib/cefs/root.py
+++ b/bin/lib/cefs/root.py
@@ -30,7 +30,7 @@ def __init__(self, *, fs_root: Path, config: CefsConfig):
         # symlink found along the path (to allow for `/some/root-owned` dir to symlink elsewhere: we actually update
         # elsewhere.
         while self._image_root.is_symlink():
-            _LOGGER.info(f"Following root symlink to {self._image_root}...")
+            _LOGGER.info("Following root symlink to %s...", self._image_root)
             self._root = self._image_root
             self._image_root = self._root.readlink()
 
diff --git a/bin/test/cefs_test.py b/bin/test/cefs_test.py
index 0e340a4fc..16a606ac4 100644
--- a/bin/test/cefs_test.py
+++ b/bin/test/cefs_test.py
@@ -6,30 +6,30 @@
 from lib.cefs.root_image import CefsRootImage, METADATA_FILENAME, BadCefsImage
 
 
-@pytest.fixture
-def cefs_mountpoint(tmp_path) -> Path:
+@pytest.fixture(name="cefs_mountpoint")
+def cefs_mountpoint_fixture(tmp_path) -> Path:
     result = tmp_path / "some-cefs-root"
     result.mkdir()
     return result
 
 
-@pytest.fixture
-def image_root(tmp_path) -> Path:
+@pytest.fixture(name="image_root")
+def image_root_fixture(tmp_path) -> Path:
     result = tmp_path / "some_image"
     result.mkdir()
     return result
 
 
-@pytest.fixture
-def dest_root(tmp_path) -> Path:
+@pytest.fixture(name="dest_root")
+def dest_root_fixture(tmp_path) -> Path:
     result = tmp_path / "dest"
     result.mkdir()
     return result
 
 
-@pytest.fixture
-def cefs_config(cefs_mountpoint, image_root) -> CefsConfig:
-    return CefsConfig(mountpoint=cefs_mountpoint, image_root=dest_root)
+@pytest.fixture(name="cefs_config")
+def cefs_config_fixture(cefs_mountpoint, image_root) -> CefsConfig:
+    return CefsConfig(mountpoint=cefs_mountpoint, image_root=image_root)
 
 
 def test_cefs_root_should_handle_no_initial_path(cefs_config):
@@ -44,7 +44,7 @@ def test_cefs_root_should_handle_empty_dirs(cefs_config: CefsConfig, image_root:
     assert not root.metadata
 
 
-def test_cefs_can_read_existing_images(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+def test_cefs_can_read_existing_images(cefs_config: CefsConfig, image_root: Path):
     (image_root / "thing").symlink_to(cefs_config.mountpoint / "thing")
     (image_root / "some" / "sub").mkdir(parents=True)
     (image_root / "some" / "sub" / "directory").symlink_to(cefs_config.mountpoint / "thing" / "in" / "subdir")
@@ -56,7 +56,7 @@ def test_cefs_can_read_existing_images(cefs_config: CefsConfig, image_root: Path
     assert not root.metadata
 
 
-def test_cefs_can_read_existing_images_metadata(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+def test_cefs_can_read_existing_images_metadata(cefs_config: CefsConfig, image_root: Path):
     (image_root / METADATA_FILENAME).write_text("Meta\nData\nFTW\n", encoding="utf-8")
     root = CefsRootImage(directory=image_root, config=cefs_config)
     assert root.metadata == ["Meta", "Data", "FTW"]
@@ -68,7 +68,7 @@ def test_cefs_is_upset_by_links_to_non_cefs(cefs_config: CefsConfig, image_root:
         CefsRootImage(directory=image_root, config=cefs_config)
 
 
-def test_cefs_is_upset_by_random_files(cefs_config: CefsConfig, image_root: Path, dest_root: Path):
+def test_cefs_is_upset_by_random_files(cefs_config: CefsConfig, image_root: Path):
     (image_root / "thing.txt").write_text("I am a mongoose", encoding="utf-8")
     with pytest.raises(BadCefsImage, match="Found an unexpected file"):
         CefsRootImage(directory=image_root, config=cefs_config)

From bb2c7c6223a25a19f761f1e9b94c22b623e1fc61 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 9 Dec 2024 14:45:23 +0000
Subject: [PATCH 155/161] Fixes to get it working

---
 bin/lib/cefs/root_image.py     | 2 +-
 bin/lib/cefs/squash.py         | 2 +-
 bin/scripts/python_module_shim | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index f9f571646..0d004bae2 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -152,7 +152,7 @@ def consolidate(self) -> None:
         self._catalog = {entry: dest_image / entry for entry, dest in self._catalog.items()}
 
     def import_existing(self, root_dir: Path, subdirs: Iterable[Path], replace: bool = False) -> None:
-        with TemporaryDirectory(prefix="cefs-import") as tmp_dir:
+        with TemporaryDirectory(prefix="cefs-import", dir=self._config.image_root) as tmp_dir:
             tmp_path = Path(tmp_dir)
             tmp_sqfs = tmp_path / "temp.sqfs"
             tmp_packfile = tmp_path / "packfile"
diff --git a/bin/lib/cefs/squash.py b/bin/lib/cefs/squash.py
index ff9e1b328..f926a3fe6 100644
--- a/bin/lib/cefs/squash.py
+++ b/bin/lib/cefs/squash.py
@@ -19,7 +19,7 @@ def __init__(self, config: CefsConfig):
 
     def import_existing_path(self, path: Path):
         assert self._sha is None
-        with TemporaryDirectory(prefix="ce-squash-builder") as tmp_dir:
+        with TemporaryDirectory(prefix="ce-squash-builder", dir=self._config.image_root) as tmp_dir:
             tmp_sqfs = Path(tmp_dir) / "temp.sqfs"
             subprocess.check_call(
                 [
diff --git a/bin/scripts/python_module_shim b/bin/scripts/python_module_shim
index 60fba5219..7abe93a24 100755
--- a/bin/scripts/python_module_shim
+++ b/bin/scripts/python_module_shim
@@ -16,5 +16,7 @@ if ! make -sq -C "${ROOT_DIR}" ce; then
     exit 1
 fi
 
+PYTHON_ENV=$(cd "${ROOT_DIR}" && "${ROOT_DIR}/.poetry/bin/poetry" env info --path)
+
 export PYTHONPATH=${ROOT_DIR}/bin
-exec "${ROOT_DIR}/.env/bin/python3" -m "lib.${MY_NAME}" "$@"
+exec "${PYTHON_ENV}/bin/python3" -m "lib.${MY_NAME}" "$@"

From 62dece11ce29cf2190d1ddc5d0c177656e1c66ba Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 9 Dec 2024 14:58:43 +0000
Subject: [PATCH 156/161] Fix symlinks, fix compression

---
 bin/lib/cefs/root_image.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index 0d004bae2..8031ed1d4 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -103,14 +103,14 @@ def _items_for(self, directory: Path, source_root: Path, dest_root: Path) -> Ite
             mode = f"0{oct(stat.st_mode & 0o7777)[2:]}"
             uid = stat.st_uid
             gid = stat.st_gid
-            if item.is_dir():
+            if item.is_symlink():
+                yield f'slink "{rel_path}" {mode} {uid} {gid} {item.readlink()}'
+            elif item.is_dir():
                 yield f'dir "{rel_path}" {mode} {uid} {gid}'
                 yield from self._items_for(directory=item, source_root=source_root, dest_root=dest_root)
             elif item.is_file():
                 # TODO better hope there's no special characters as gensquashfs won't let the <location> have any...
                 yield f'file "{rel_path}" {mode} {uid} {gid} {item}'
-            elif item.is_symlink():
-                yield f'slink "{rel_path}" {mode} {uid} {gid} {item.readlink()}'
             else:
                 raise RuntimeError(f"oh no {item}")  # TODO
 
@@ -138,6 +138,8 @@ def consolidate(self) -> None:
                     str(tmp_packfile),
                     "--compressor",
                     "zstd",
+                    "-X",
+                    "level=19",
                 ]
             )
             sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()
@@ -185,6 +187,8 @@ def import_existing(self, root_dir: Path, subdirs: Iterable[Path], replace: bool
                     str(tmp_packfile),
                     "--compressor",
                     "zstd",
+                    "-X",
+                    "level=19",
                 ]
             )
             sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()

From e0b1eedc697d21b23f43ec72d795e22eb7b8b4b5 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 9 Dec 2024 15:03:53 +0000
Subject: [PATCH 157/161] Reinstate

---
 bin/yaml/libraries.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index 228c537a6..a0fd252cf 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -2747,6 +2747,11 @@ libraries:
       - 3.1.18
       - 4.5.20
       type: cratesio
+    color-eyre:
+      build_type: cargo
+      targets:
+        - 0.6.2
+      type: cratesio
     contracts:
       build_type: cargo
       targets:

From 62aa0339f7adb8efa38ca16c7bbd30374ac9a9dc Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Mon, 9 Dec 2024 15:18:22 +0000
Subject: [PATCH 158/161] Olafur's yellow box

---
 bin/lib/installable/installable.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/lib/installable/installable.py b/bin/lib/installable/installable.py
index 572b17271..8c06f7084 100644
--- a/bin/lib/installable/installable.py
+++ b/bin/lib/installable/installable.py
@@ -292,9 +292,10 @@ def squash_to(self, destination_image: Path):
         self._logger.info("Squashing %s...", source_folder)
         self.install_context.check_call(
             [
-                "mksquashfs",
+                "/usr/bin/mksquashfs",
                 str(source_folder),
                 str(temp_image),
+                "-all-root",
                 "-progress",
                 "-comp",
                 "zstd",

From 0e6b00175dada62bec5d309e395046088431dd85 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Tue, 10 Dec 2024 08:52:33 +0000
Subject: [PATCH 159/161] zomg

---
 bin/yaml/libraries.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/yaml/libraries.yaml b/bin/yaml/libraries.yaml
index a0fd252cf..36953a876 100644
--- a/bin/yaml/libraries.yaml
+++ b/bin/yaml/libraries.yaml
@@ -2750,7 +2750,7 @@ libraries:
     color-eyre:
       build_type: cargo
       targets:
-        - 0.6.2
+      - 0.6.2
       type: cratesio
     contracts:
       build_type: cargo

From 5bd76c5fe16be9a7be9a3cb37c9b4c81a4e02551 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Thu, 12 Dec 2024 15:28:48 +0000
Subject: [PATCH 160/161] Use new way of linking anything in /cefs/bla/fah/shah
 to /image-root/bla/fa/shah.sqfs. cc @amorton

---
 bin/lib/cefs/__main__.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index 99be35097..aa65999ac 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -84,8 +84,27 @@ def install(context: CliContext):
     click.echo("Writing config files...")
     auto_cefs_config_file = Path("/etc/auto.cefs")
     auto_cefs_config_file.write_text(
-        f"* -fstype=squashfs,loop,nosuid,nodev,ro :{config.image_root}/&.sqfs\n", encoding="utf-8"
+        f"""#!/bin/bash
+key="$1"
+# Abuse the fact PWD is the parent dir to derive the full path to this subdirectory.
+SUBDIR=${{PWD#{config.mountpoint}}}
+if [[ ! "$SUBDIR" = "#{config.mountpoint}" ]]; then
+   key=$SUBDIR/$key
+fi
+# Remove any trailing slash
+key=${{key%/}}
+# If it's a sqfs image, then mount it.
+if [[ -f "{config.image_root}/$key.sqfs" ]]; then
+    echo "-fstype=squashfs,loop,nosuid,nodev,ro :{config.image_root}/$key.sqfs"
+fi
+# Else if it's the name of a directory, make it appear as another autofs level.
+if [[ -d "{config.image_root}/$key" ]]; then
+    echo "-fstype=autofs :{auto_cefs_config_file}"
+fi
+""",
+        encoding="utf-8",
     )
+    auto_cefs_config_file.chmod(0o700)
     Path("/etc/auto.master.d/cefs.autofs").write_text(
         f"{config.mountpoint} {auto_cefs_config_file}  --negative-timeout 1", encoding="utf-8"
     )

From b8ad4b721d14c9f355bd6de47fea93dcba4d6850 Mon Sep 17 00:00:00 2001
From: Matt Godbolt <matt@godbolt.org>
Date: Thu, 12 Dec 2024 15:59:38 +0000
Subject: [PATCH 161/161] More CEFS note taking

---
 bin/lib/cefs/__main__.py   | 1 +
 bin/lib/cefs/root_image.py | 7 +++++++
 bin/lib/cefs/squash.py     | 7 +++++++
 3 files changed, 15 insertions(+)

diff --git a/bin/lib/cefs/__main__.py b/bin/lib/cefs/__main__.py
index aa65999ac..af81c8b11 100644
--- a/bin/lib/cefs/__main__.py
+++ b/bin/lib/cefs/__main__.py
@@ -262,6 +262,7 @@ def import_(context: CliContext, root: Path, relative_to: Path, directory: List[
 # TODO consolidation in general! aim is to have very few layers, after all.
 # TODO delete old things
 # TODO keep track of a symlink and its history /opt/compiler-explorer --> /opt/compiler-explorer.history?
+# .... or ln -sf .prev to the previous state
 # TODO better naming of dirs/manifest for each so we know who and why created each layer?
 # TODO should we separate out root squashfs images from data?
 # TODO detect cefs roots in `install` and redirect to buildroot?
diff --git a/bin/lib/cefs/root_image.py b/bin/lib/cefs/root_image.py
index 8031ed1d4..9aa42c658 100644
--- a/bin/lib/cefs/root_image.py
+++ b/bin/lib/cefs/root_image.py
@@ -142,10 +142,14 @@ def consolidate(self) -> None:
                     "level=19",
                 ]
             )
+            # todo reconcile with other code elsehwere and proper naming. probably move things into self._config to
+            # get named parts?
             sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            # todo image_root/data/parts/sha/here.sqfs ?
             image = self._config.image_root / f"{sha}.sqfs"
             if not image.exists():
                 _LOGGER.info("New squashfs image: %s", image)
+                image.parent.mkdir(parents=True, exist_ok=True)
                 tmp_sqfs.replace(image)
             else:
                 _LOGGER.info("Existing: %s", image)
@@ -191,10 +195,13 @@ def import_existing(self, root_dir: Path, subdirs: Iterable[Path], replace: bool
                     "level=19",
                 ]
             )
+            # todo de-duplicate with the other similar
             sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            # todo namessss like image_root/data/sha/parts/here.sqfs
             image = self._config.image_root / f"{sha}.sqfs"
             if not image.exists():
                 _LOGGER.info("New squashfs image: %s", image)
+                image.parent.mkdir(parents=True, exist_ok=True)
                 tmp_sqfs.replace(image)
             else:
                 _LOGGER.info("Existing: %s", image)
diff --git a/bin/lib/cefs/squash.py b/bin/lib/cefs/squash.py
index f926a3fe6..21f7a8fff 100644
--- a/bin/lib/cefs/squash.py
+++ b/bin/lib/cefs/squash.py
@@ -12,6 +12,7 @@
 _LOGGER = logging.getLogger(__name__)
 
 
+# TODO work out wth this is and why we need it separately from the rest
 class SquashFsCreator:
     def __init__(self, config: CefsConfig):
         self._config = config
@@ -26,16 +27,22 @@ def import_existing_path(self, path: Path):
                     "mksquashfs",
                     str(path),
                     str(tmp_sqfs),
+                    # todo - force root?
                     "-root-mode",
                     "755",
                     "-progress",
                     "-comp",
                     "zstd",
+                    # todo compression ratio
                 ]
             )
+            # todo de-duplicate with the other similar
             self._sha, _filename = subprocess.check_output(["shasum", str(tmp_sqfs)]).decode("utf-8").split()
+            # todo rename sha as path part? split along the name sh/a/thing.sqfs
+            # todo consider a "root" so it's actually image_root/images/sha/parts/here.sqfs
             if not self.image.exists():
                 _LOGGER.info("New squashfs image: %s", self.image)
+                self.image.parent.mkdir(parents=True, exist_ok=True)
                 tmp_sqfs.replace(self.image)
             else:
                 _LOGGER.info("Existing: %s", self.image)