Add APT proxy and document use case

.github/workflows/integration-test.yml

@@ -61,6 +61,23 @@ jobs:
       - name: Compile action code before test
         run: npm ci && npm run package
 
+      - name: Restore package cache
+        uses: actions/cache@v4
+        with:
+          path: apt-cache/
+          key: ${{ runner.os }}-${{ github.job }}
+          restore-keys: |
+            ${{ runner.os }}
+
+      - name: Setup APT proxy on runner
+        run: |
+          mkdir -p apt-cache
+          sudo apt-get install -y apt-cacher-ng --no-install-suggests --no-install-recommends
+          sudo bash -c 'echo -e "Port: 9999\nDebug: 99" >> /etc/apt-cacher-ng/acng.conf'
+          sudo mv -f apt-cache/* /var/cache/apt-cacher-ng/ || true
+          sudo chown -R apt-cacher-ng:apt-cacher-ng /var/cache/apt-cacher-ng
+          sudo service apt-cacher-ng restart
+
       - name: Run pi-gen build
         uses: ./
         id: build
@@ -81,6 +98,11 @@ jobs:
           timezone: ${{ env.CONFIG_TIMEZONE }}
           pubkey-ssh-first-user: ${{ env.CONFIG_PUBLIC_KEY }}
           increase-runner-disk-size: ${{ github.event_name != 'workflow_dispatch' || inputs.increase-runner-disk }}
+          apt-proxy: http://172.17.0.1:9999
+
+      - name: Move APT cache
+        continue-on-error: true
+        run: rm -rf apt-cache/* && sudo mv -f /var/cache/apt-cacher-ng/* apt-cache/
 
       - name: List working directory
         run: tree 
@@ -112,4 +134,11 @@ jobs:
         with:
           labels: test
 
+      - name: Debug APT proxy
+        if: always()
+        run: |
+          sudo tail -n 100 /etc/apt-cacher-ng/acng.conf
+          sudo service apt-cacher-ng status
+          sudo tail -n 200 /var/log/syslog /var/log/apt-cacher-ng/* 
+
 

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Debug Jest Tests",
+        "type": "node",
+        "request": "launch",
+        "runtimeArgs": [
+          "--inspect-brk",
+          "${workspaceRoot}/node_modules/.bin/jest",
+          "--runInBand"
+        ],
+        "console": "integratedTerminal",
+        "internalConsoleOptions": "neverOpen"
+      }
+    ]
+  }

README.md

@@ -55,6 +55,11 @@ tries to make sure the stage is respected and its changes are included in the fi
 ```yaml
 - uses: usimd/pi-gen-action@v1
   with:
+    # If you require the use of an apt proxy, set it here. This proxy setting will not 
+    # be included in the image, making it safe to use an apt-cacher or similar package 
+    # for development.
+    apt-proxy: ''
+
     # Compression to apply on final image (either "none", "zip", "xz" or "gz").
     compression: zip
 
@@ -197,6 +202,7 @@ tries to make sure the stage is respected and its changes are included in the fi
 - [Upload final image as artifact](#upload-final-image-as-artifact)
 - [Modify `pi-gen` internal stages](#modify-pi-gen-internal-stages)
 - [Increase GitHub Actions runner disk space](#increase-github-actions-runner-disk-space)
+- [Use an APT proxy to define where to pull packages from](#use-fast-apt-proxy-for-pi-gen)
 
 ### Install NodeJS from Nodesource in the target image
 ```yaml
@@ -302,7 +308,7 @@ runs the `pi-gen` build always inside a container).
 
 From current experience, this will reclaim between 25 and 30 GB of additional disk space
 
-```
+```yaml
 jobs:
   pi-gen-with-larger-disk-space:
     runs-on: ubuntu-latest
@@ -314,6 +320,56 @@ jobs:
           increase-runner-disk-size: true
 ```
 
+### Use APT proxy for `pi-gen`
+
+There might be scenarios where you need to specifically select the package
+mirror to pull from:
+* custom runners with limited network access
+* improved download speed when selecting a fast mirror via CDN does not suffice
+* speed up build by caching pulled packages on the runner
+
+The latter is sketched below by adding `apt-cacher-ng` to the runner where its
+cache directory is stored as a GitHub Actions cache. If you want to set up such
+a scenario, remember to tweak the caching behavior of `apt-cacher-ng` so it
+can make best use of the restored cache.
+
+```yaml
+jobs:
+  pi-gen-with-apt-proxy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Restore package cache
+        uses: actions/cache@v4
+        with:
+          # Unpacking directly into /var/cache would result in permission errors
+          # so we need to work around this a bit.
+          path: apt-cache/
+          key: ${{ runner.os }}-${{ github.job }}
+          restore-keys:
+            - ${{ runner.os }}
+
+      - name: Setup APT proxy on runner
+        run: |
+          mkdir -p apt-cache
+          sudo apt-get install -y apt-cacher-ng --no-install-suggests --no-install-recommends
+          sudo bash -c 'echo "Port: 9999" >> /etc/apt-cacher-ng/acng.conf'
+          sudo mv -f apt-cache/* /var/cache/apt-cacher-ng/ || true
+          sudo chown -R apt-cacher-ng:apt-cacher-ng /var/cache/apt-cacher-ng
+          sudo service apt-cacher-ng restart
+
+      - uses: usimd/pi-gen-action@v1
+        with:
+          image-name: test
+          stage-list: stage0 stage1 stage2 custom-stage
+          # Instead of referring to the runner's container IP directly, you could also add
+          # a new host entry to the pi-gen container:
+          # docker-opts: --add-host=host.docker.internal:host-gateway
+          apt-proxy: http://172.17.0.1:9999
+
+      - name: Move packages to temp location for caching
+        run: sudo mv -f /var/cache/apt-cache-ng/* apt-cache/
+```
+
 ## License
 
 The scripts and documentation in this project are released under the [MIT License](LICENSE)

__test__/pi-gen-config.test.ts

@@ -141,6 +141,11 @@ describe('PiGenConfig', () => {
       'pubkeySshFirstUser',
       'ssh-foo vnqf493rn34xzrm234yru13ß48rnz1x034ztn== [email protected]',
       'pubkey-ssh-first-user does not seem to be a valid list of public key according to "ssh-keygen -l", here\'s its output'
+    ],
+    [
+      'aptProxy',
+      'this/is/not/valid',
+      'apt-proxy is not a valid URL. Make it point to a correct http/https address'
     ]
   ])(
     'rejects %s with invalid value %s',
@@ -152,9 +157,9 @@ describe('PiGenConfig', () => {
       expect(await validateConfig(piGenConfig)).toBeUndefined()
 
       piGenConfig[property as keyof PiGenConfig] = value
-      expect(async () => await validateConfig(piGenConfig)).rejects.toThrow(
-        error
-      )
+      await expect(
+        async () => await validateConfig(piGenConfig)
+      ).rejects.toThrow(error)
     }
   )
 

action.yml

@@ -4,7 +4,12 @@ author: Simon Domke
 
 inputs:
   # pi-gen variables
-
+  apt-proxy:
+    description: |
+      If you require the use of an apt proxy, set it here. This proxy setting will not be included in the image,
+      making it safe to use an apt-cacher or similar package for development.
+    required: false
+    default: ''
   image-name:
     description: Final image name.
     required: true

src/configure.ts

@@ -56,6 +56,7 @@ export async function configure(): Promise<PiGenConfig> {
     userConfig.setfcap = core.getInput('setfcap') || DEFAULT_CONFIG.setfcap
     userConfig.piGenRelease =
       core.getInput('pi-gen-release') || DEFAULT_CONFIG.piGenRelease
+    userConfig.aptProxy = core.getInput('apt-proxy') || DEFAULT_CONFIG.aptProxy
 
     await validateConfig(userConfig)
 

src/pi-gen-config.ts

@@ -6,6 +6,7 @@ import * as exec from '@actions/exec'
 import * as io from '@actions/io'
 
 export interface PiGenConfig {
+  aptProxy?: string
   imgName: string
   piGenRelease: string
   release: string
@@ -220,6 +221,16 @@ export async function validateConfig(config: PiGenConfig): Promise<void> {
       }
     }
   }
+
+  if (config.aptProxy) {
+    try {
+      new URL(config.aptProxy)
+    } catch (error) {
+      throw new Error(
+        'apt-proxy is not a valid URL. Make it point to a correct http/https address'
+      )
+    }
+  }
 }
 
 function camelCaseToSnakeCase(label: string): string {