CI: Improve prebuilt/build checking

shajra · Jul 14, 2024 · 585a247 · 585a247
1 parent 02ea8e7
commit 585a247
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 92 deletions.
diff --git a/build/overlay.nix b/build/overlay.nix
@@ -63,11 +63,8 @@ let
 
     pkgs.lists = deepDrvSetToList pkgs.sets;
 
-    checkPrebuilt =
+    checkCaching = sets: checkCacheHit:
         let
-            sets = build.pkgs.nixpkgs.prebuilt
-                // build.pkgs.haskell-nix.prebuilt
-                // build.pkgs.shajra.prebuilt;
             found = builtins.attrValues (deepMerge (lib.collect isDrvSet sets));
             bashOutPaths = lib.concatMapStringsSep " " (d: d.outPath) found;
         in prev.writeShellApplication {
@@ -81,21 +78,38 @@ let
                 set -o pipefail
                 OUT_PATHS=(${bashOutPaths})
                 EXIT_CODE=0
+                echo "${
+                    if checkCacheHit
+                    then "Checking prebuilt packages cached..."
+                    else "Checking packages to build uncached..."
+                }"
                 for path in "''${OUT_PATHS[@]}"
                 do
                     PATH_PREFIX="''${path%%-*}"
                     HASH="$(basename "$PATH_PREFIX")"
-                    if ! curl --fail --silent --head \
+                    if ${ if checkCacheHit then "!" else "" } curl --fail --silent --head \
                         "https://cache.nixos.org/$HASH.narinfo" >/dev/null
                     then
-                        echo "NOT FOUND: $path"
+                        echo "${ if checkCacheHit then "NOT" else "" } FOUND: $path"
                         EXIT_CODE=1
                     fi
                 done
                 exit "$EXIT_CODE"
             '';
         };
 
+    checkPrebuilt =
+        let sets = build.pkgs.nixpkgs.prebuilt
+                // build.pkgs.haskell-nix.prebuilt
+                // build.pkgs.shajra.prebuilt;
+        in checkCaching sets true;
+
+    checkBuild =
+        let sets = build.pkgs.nixpkgs.build
+                // build.pkgs.haskell-nix.build
+                // build.pkgs.shajra.build;
+        in checkCaching sets false;
+
     ci.prebuilt.nixpkgs.lower  = joinForCi "prebuilt-nixpkgs"    isLower  build.pkgs.nixpkgs.prebuilt;
     ci.prebuilt.nixpkgs.middle = joinForCi "prebuilt-nixpkgs"    isMiddle build.pkgs.nixpkgs.prebuilt;
     ci.prebuilt.nixpkgs.upper  = joinForCi "prebuilt-nixpkgs"    isUpper  build.pkgs.nixpkgs.prebuilt;
@@ -106,6 +120,7 @@ let
     ci.build.shajra            = joinForCi "build-shajra"        isAny    build.pkgs.shajra.build;
     ci.all                     = joinForCi "all"                 isAny    build.pkgs;
     ci.check-prebuilt = checkPrebuilt;
+    ci.check-build = checkBuild;
 
 in {
     shajra-provision = {

diff --git a/doc/nix-usage-flakes.md b/doc/nix-usage-flakes.md
@@ -113,10 +113,8 @@ nix flake show .
     ├───apps
     │   ├───aarch64-darwin
     …
-        │   ├───ci-build-nixpkgs omitted (use '--all-systems' to show)
-        │   ├───ci-build-shajra omitted (use '--all-systems' to show)
-        │   ├───ci-prebuilt-haskellnix omitted (use '--all-systems' to show)
-        │   ├───ci-prebuilt-nixpkgs omitted (use '--all-systems' to show)
+        │   ├───ci-prebuilt-nixpkgs-middle omitted (use '--all-systems' to show)
+        │   ├───ci-prebuilt-nixpkgs-upper omitted (use '--all-systems' to show)
         │   ├───ci-prebuilt-shajra omitted (use '--all-systems' to show)
         │   ├───home-manager omitted (use '--all-systems' to show)
         │   ├───shajra-darwin-rebuild omitted (use '--all-systems' to show)
@@ -127,7 +125,9 @@ nix flake show .
             ├───ci-build-nixpkgs: package 'shajra-provision-ci-build-nixpkgs'
             ├───ci-build-shajra: package 'shajra-provision-ci-build-shajra'
             ├───ci-prebuilt-haskellnix: package 'shajra-provision-ci-prebuilt-haskellnix'
-            ├───ci-prebuilt-nixpkgs: package 'shajra-provision-ci-prebuilt-nixpkgs'
+            ├───ci-prebuilt-nixpkgs-lower: package 'shajra-provision-ci-prebuilt-nixpkgs'
+            ├───ci-prebuilt-nixpkgs-middle: package 'shajra-provision-ci-prebuilt-nixpkgs'
+            ├───ci-prebuilt-nixpkgs-upper: package 'shajra-provision-ci-prebuilt-nixpkgs'
             ├───ci-prebuilt-shajra: package 'shajra-provision-ci-prebuilt-shajra'
             ├───home-manager: package 'home-manager'
             ├───shajra-darwin-rebuild: package 'shajra-darwin-rebuild'
@@ -186,7 +186,7 @@ nix search .
 
     * packages.x86_64-linux.ci-prebuilt-haskellnix
 
-    * packages.x86_64-linux.ci-prebuilt-nixpkgs
+    * packages.x86_64-linux.ci-prebuilt-nixpkgs-lower
     …
 
 If a flake has a lot of packages, you can pass regexes to prune down the search. Returned values will match all the regexes provided.

diff --git a/doc/nix-usage-flakes.org b/doc/nix-usage-flakes.org
@@ -203,10 +203,8 @@ git+file:///home/shajra/src/shajra-provisioning
 ├───apps
 │   ├───aarch64-darwin
 …
-    │   ├───ci-build-nixpkgs omitted (use '--all-systems' to show)
-    │   ├───ci-build-shajra omitted (use '--all-systems' to show)
-    │   ├───ci-prebuilt-haskellnix omitted (use '--all-systems' to show)
-    │   ├───ci-prebuilt-nixpkgs omitted (use '--all-systems' to show)
+    │   ├───ci-prebuilt-nixpkgs-middle omitted (use '--all-systems' to show)
+    │   ├───ci-prebuilt-nixpkgs-upper omitted (use '--all-systems' to show)
     │   ├───ci-prebuilt-shajra omitted (use '--all-systems' to show)
     │   ├───home-manager omitted (use '--all-systems' to show)
     │   ├───shajra-darwin-rebuild omitted (use '--all-systems' to show)
@@ -217,7 +215,9 @@ git+file:///home/shajra/src/shajra-provisioning
         ├───ci-build-nixpkgs: package 'shajra-provision-ci-build-nixpkgs'
         ├───ci-build-shajra: package 'shajra-provision-ci-build-shajra'
         ├───ci-prebuilt-haskellnix: package 'shajra-provision-ci-prebuilt-haskellnix'
-        ├───ci-prebuilt-nixpkgs: package 'shajra-provision-ci-prebuilt-nixpkgs'
+        ├───ci-prebuilt-nixpkgs-lower: package 'shajra-provision-ci-prebuilt-nixpkgs'
+        ├───ci-prebuilt-nixpkgs-middle: package 'shajra-provision-ci-prebuilt-nixpkgs'
+        ├───ci-prebuilt-nixpkgs-upper: package 'shajra-provision-ci-prebuilt-nixpkgs'
         ├───ci-prebuilt-shajra: package 'shajra-provision-ci-prebuilt-shajra'
         ├───home-manager: package 'home-manager'
         ├───shajra-darwin-rebuild: package 'shajra-darwin-rebuild'
@@ -308,7 +308,7 @@ nix search . | ansifilter
 
 ,* packages.x86_64-linux.ci-prebuilt-haskellnix
 
-,* packages.x86_64-linux.ci-prebuilt-nixpkgs
+,* packages.x86_64-linux.ci-prebuilt-nixpkgs-lower
 …
 #+end_example
 

diff --git a/home/modules/base/tui/all/programs/emacs/default.nix b/home/modules/base/tui/all/programs/emacs/default.nix
@@ -7,6 +7,11 @@ pkgs: {
         vterm
     ];
 
+    # REVISIT: emacs-macport was broken for M1
+    # https://github.com/NixOS/nixpkgs/issues/127902
+    # REVISIT: note emacs-macport doesn't have native compilation
+    # https://github.com/railwaycat/homebrew-emacsmacport/issues/274
+
     # DESIGN: https://github.com/doomemacs/doomemacs#prerequisites
     # using emacs29 over emacs-unstable to hit Nixpkgs cache
     package = pkgs.emacs29;

diff --git a/packages.nix b/packages.nix
@@ -34,31 +34,23 @@ in {
     ];
 
     nixpkgs.prebuilt.base.gui.all = pickHome [
-        # Fonts
+        # Cached Fonts
         "emacs-all-the-icons-fonts"  # for Emacs, used automatically by Doom
         "etBook"                     # stylish font from Edward Tufte's books
         "fira"                       # variable font to complement Fira Code
         "font-awesome_6"             # for i3status-rust icons
         "freefont_ttf"               # a Unicode fallback font
         "hasklig"                    # font with Haskell ligatures
-        "inconsolata-nerdfont"       # popular font, might like it
-        "nerdfonts"                  # developer fonts with lots of icons
         "noto-fonts-emoji"           # popular font, might like it
         "noto-fonts-extra"           # popular font, might like it
         "noto-fonts"                 # popular font, might like it
-        "sf-symbols"                 # font with application icons
-        "sketchybar-font"            # font with application icons
         "source-serif"               # serif font to complement Sauce Code Pro
-        "symbola"                    # another Unicode fallback font
     ];
 
     nixpkgs.prebuilt.base.gui.darwin = np.pick { darwin = "home"; } [
     ] // np.pick { darwin = "unstable"; } [
         # DESIGN: these are all Mac-only applications; unstable is fine
-        "aldente"
         "goku"
-        "raycast"
-        "shortcat"
     ] // np.pick { darwin = "stable-linux"; } [
         # DESIGN: iTerm2 3.5 messes up colors with -CC
         # TODO: Search for problem in iTerm's issue tracker
@@ -71,7 +63,6 @@ in {
         "fontpreview"
         "gnome.adwaita-icon-theme"
         "maim"
-        "microsoft-edge-beta"
         "pavucontrol"
         "simple-scan"
         "sxiv"
@@ -109,7 +100,6 @@ in {
         "nixfmt"
         "paperkey"
         "patchelf"
-        "preview-file"
         "procps"
         "pstree"
         "rsync"
@@ -121,11 +111,7 @@ in {
         "wget"
         "which"
         "yq-go"
-    ] // {
-        aspell = np.nixpkgs.home.aspellWithDicts (d: with d; [
-            en en-computers en-science
-        ]);
-    };
+    ];
 
     nixpkgs.prebuilt.base.tui.darwin = np.pick { darwin = "home"; } [
     ] // np.pick { darwin = "unstable"; } [
@@ -148,12 +134,6 @@ in {
         "bukubrow"
     ];
 
-    nixpkgs.prebuilt.chat.gui.all = pickHome [
-        "discord"
-        "slack"
-        "zoom-us"
-    ];
-
     nixpkgs.prebuilt.chat.gui.linux = np.pick { linux = "home"; } [
         "element-desktop"
         "irccloud"
@@ -245,31 +225,55 @@ in {
 
     nixpkgs.build.base.gui.all = pickHome [
         "notify-time"
+
+        # Uncached Fonts
+        "inconsolata-nerdfont"       # popular font, might like it
+        "nerdfonts"                  # developer fonts with lots of icons
+        "sf-symbols"                 # font with application icons
+        "sketchybar-font"            # font with application icons
+        "symbola"                    # another Unicode fallback font
     ];
 
     nixpkgs.build.base.gui.darwin = np.pick { darwin = "home"; } [
         # REVISIT: Waiting for Nixpkgs to have Mac SDK 14
         #"yabai"
         #"skhd"
+    ] // np.pick { darwin = "unstable"; } [
+        # DESIGN: these are all Mac-only applications; unstable is fine
+        "aldente"
+        "raycast"
+        "shortcat"
     ];
 
     nixpkgs.build.base.gui.linux = np.pick { linux = "home"; } [
         "dunst-osd"
         "i3-dpi"
         "i3status-rust-dunst"
         "i3-workspace-name"
+        "microsoft-edge-beta"
     ];
 
     nixpkgs.build.base.tui.all = pickHome [
-        "home-manager"
-    ];
+        "home-manager-latest"
+        "preview-file"
+    ] // {
+        aspell = np.nixpkgs.home.aspellWithDicts (d: with d; [
+            en en-computers en-science
+        ]);
+    };
 
     nixpkgs.build.base.tui.darwin = np.pick { darwin = "home"; } [
     ];
 
     nixpkgs.build.base.tui.linux = np.pick { linux = "home"; } [
     ];
 
+    nixpkgs.build.chat.gui.all = pickHome [
+        "discord"
+        "slack"
+        "zoom-us"
+    ];
+
     nixpkgs.build.finance = pickHome [
         "moneydance"
     ];
@@ -304,19 +308,11 @@ in {
     ];
 
     nixpkgs.build.uncategorized.darwin = np.pick { darwin = "home"; } [
-        # REVISIT: emacsMacport broken for M1
-        # https://github.com/NixOS/nixpkgs/issues/127902
-        # REVISIT: note emacsMacport doesn't have native compilation
-        # https://github.com/railwaycat/homebrew-emacsmacport/issues/274
-        #"emacsMacport"
         "sketchybar-cpu"
         "sketchybar-lua"
     ];
 
     nixpkgs.build.uncategorized.linux = np.pick { linux = "home"; } [
-        # REVISIT: https://github.com/doomemacs/doomemacs#prerequisites
-        # using emacs29 over emacs-unstable to hit Nixpkgs cache
-        "emacs29"
     ];
 
     haskell-nix.prebuilt.programming.haskell = {

diff --git a/support/packages-cached b/support/packages-cached
diff --git a/support/packages-check b/support/packages-check
@@ -0,0 +1,6 @@
+#!/bin/sh -eu
+
+cd "$(dirname "$0")/.."
+
+nix run .#ci.check-prebuilt
+nix run .#ci.check-build