Add APT proxy and document use case

.github/workflows/integration-test.yml

@@ -61,6 +61,34 @@ jobs:
       - name: Compile action code before test
         run: npm ci && npm run package
 
+      - name: Select Debian package mirror
+        id: select-mirror
+        # This is just to demonstrate that you could determine the mirror to use
+        # however you fancy.
+        run: echo "mirror=debian-archive.trafficmanager.net" >> $GITHUB_OUTPUT
+
+      - name: Setup APT proxy on runner
+        run: |
+          sudo bash -c 'cat > /etc/nginx/sites-enabled/default <<-"EOF"
+          server {
+            listen               9999;
+            access_log           /var/log/nginx/cache-access.log;
+            error_log            /var/log/nginx/cache-error.log;
+            allow                all;
+            resolver             8.8.8.8;
+            location / {
+              proxy_set_header   Host $host;
+              proxy_set_header   X-Forwarded-For $proxy_add_x_forwarded_for;
+              if ($host ~* ^deb\.debian\.org) {
+                proxy_pass       $scheme://${{ steps.select-mirror.outputs.mirror }}$request_uri;
+                break;
+              }
+              proxy_pass         $scheme://$host$request_uri;
+            }
+          }
+          EOF'
+          sudo service nginx restart
+
       - name: Run pi-gen build
         uses: ./
         id: build
@@ -81,6 +109,8 @@ jobs:
           timezone: ${{ env.CONFIG_TIMEZONE }}
           pubkey-ssh-first-user: ${{ env.CONFIG_PUBLIC_KEY }}
           increase-runner-disk-size: ${{ github.event_name != 'workflow_dispatch' || inputs.increase-runner-disk }}
+          docker-opts: --add-host=host.docker.internal:host-gateway
+          apt-proxy: http://host.docker.internal:9999
 
       - name: List working directory
         run: tree 
@@ -112,4 +142,11 @@ jobs:
         with:
           labels: test
 
+      - name: Debug APT proxy
+        if: always()
+        run: |
+          sudo tail -n 100 /etc/nginx/sites-enabled/default
+          sudo service nginx status
+          sudo tail -n 200 /var/log/syslog /var/log/nginx/* 
+
           

README.md

@@ -55,6 +55,11 @@ tries to make sure the stage is respected and its changes are included in the fi
 ```yaml
 - uses: usimd/pi-gen-action@v1
   with:
+    # If you require the use of an apt proxy, set it here. This proxy setting will not 
+    # be included in the image, making it safe to use an apt-cacher or similar package 
+    # for development.
+    apt-proxy: ''
+
     # Compression to apply on final image (either "none", "zip", "xz" or "gz").
     compression: zip
 
@@ -197,6 +202,7 @@ tries to make sure the stage is respected and its changes are included in the fi
 - [Upload final image as artifact](#upload-final-image-as-artifact)
 - [Modify `pi-gen` internal stages](#modify-pi-gen-internal-stages)
 - [Increase GitHub Actions runner disk space](#increase-github-actions-runner-disk-space)
+- [Use an APT proxy to define where to pull packages from](#use-fast-apt-proxy-for-pi-gen)
 
 ### Install NodeJS from Nodesource in the target image
 ```yaml
@@ -314,6 +320,58 @@ jobs:
           increase-runner-disk-size: true
 ```
 
+### Use fast APT proxy for `pi-gen`
+
+If you want to speed up your build by improving the package download speed, you can setup
+a local APT proxy and let `pi-gen` use it during the build. The proxy should point to a
+fast mirror, on GitHub Actions runners this will _very_ likely be `debian-archive.trafficmanager.net`
+hosted on Azure. Make sure, though, that the selected mirror contains your targeted
+architecture (`debian-archive.trafficmanager.net` does **not** include `armhf`).
+
+This might also be interesting for anyone setting up custom runners and trying to improve
+package pull (think about the caching possibilities):
+
+```
+jobs:
+  pi-gen-with-fast-apt-proxy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Select Debian package mirror
+        id: select-mirror
+        # This is just to demonstrate that you could determine the mirror to use
+        # however you fancy.
+        run: echo "mirror=debian-archive.trafficmanager.net" >> $GITHUB_OUTPUT
+
+      - name: Setup APT proxy on runner
+        run: |
+          sudo bash -c 'cat > /etc/nginx/sites-enabled/default <<-"EOF"
+          server {
+            listen          9999;
+            access_log      /var/log/nginx/cache-access.log;
+            error_log       /var/log/nginx/cache-error.log;
+            allow           all;
+            resolver        8.8.8.8;
+            location / {
+              if ($host ~* ^deb\.debian\.org) {
+                proxy_pass  $scheme://${{ steps.select-mirror.outputs.mirror }}$request_uri;
+                break;
+              }
+              proxy_pass    $scheme://$host$request_uri;
+            }
+          }
+          EOF'
+          sudo service nginx restart
+
+      - uses: usimd/pi-gen-action@v1
+        with:
+          image-name: test
+          stage-list: stage0 stage1 stage2 custom-stage
+          pi-gen-dir: ${{ inputs.custom-pi-gen-dir }}
+          # This is important: make the host service available to the pi-gen container
+          docker-opts: --add-host=host.docker.internal:host-gateway
+          apt-proxy: http://host.docker.internal:9999
+```
+
 ## License
 
 The scripts and documentation in this project are released under the [MIT License](LICENSE)

__test__/pi-gen-config.test.ts

@@ -141,6 +141,11 @@ describe('PiGenConfig', () => {
       'pubkeySshFirstUser',
       'ssh-foo vnqf493rn34xzrm234yru13ß48rnz1x034ztn== [email protected]',
       'pubkey-ssh-first-user does not seem to be a valid list of public key according to "ssh-keygen -l", here\'s its output'
+    ],
+    [
+      'aptProxy',
+      'this/is/not/valid',
+      'apt-proxy is not a valid URL. Make it point to a correct http/https address'
     ]
   ])(
     'rejects %s with invalid value %s',
@@ -152,9 +157,9 @@ describe('PiGenConfig', () => {
       expect(await validateConfig(piGenConfig)).toBeUndefined()
 
       piGenConfig[property as keyof PiGenConfig] = value
-      expect(async () => await validateConfig(piGenConfig)).rejects.toThrow(
-        error
-      )
+      await expect(
+        async () => await validateConfig(piGenConfig)
+      ).rejects.toThrow(error)
     }
   )
 

action.yml

@@ -4,7 +4,12 @@ author: Simon Domke
 
 inputs:
   # pi-gen variables
-
+  apt-proxy:
+    description: |
+      If you require the use of an apt proxy, set it here. This proxy setting will not be included in the image,
+      making it safe to use an apt-cacher or similar package for development.
+    required: false
+    default: ''
   image-name:
     description: Final image name.
     required: true

src/configure.ts

@@ -56,6 +56,7 @@ export async function configure(): Promise<PiGenConfig> {
     userConfig.setfcap = core.getInput('setfcap') || DEFAULT_CONFIG.setfcap
     userConfig.piGenRelease =
       core.getInput('pi-gen-release') || DEFAULT_CONFIG.piGenRelease
+    userConfig.aptProxy = core.getInput('apt-proxy') || DEFAULT_CONFIG.aptProxy
 
     await validateConfig(userConfig)
 

src/pi-gen-config.ts

@@ -6,6 +6,7 @@ import * as exec from '@actions/exec'
 import * as io from '@actions/io'
 
 export interface PiGenConfig {
+  aptProxy?: string
   imgName: string
   piGenRelease: string
   release: string
@@ -220,6 +221,16 @@ export async function validateConfig(config: PiGenConfig): Promise<void> {
       }
     }
   }
+
+  if (config.aptProxy) {
+    try {
+      new URL(config.aptProxy)
+    } catch (error) {
+      throw new Error(
+        'apt-proxy is not a valid URL. Make it point to a correct http/https address'
+      )
+    }
+  }
 }
 
 function camelCaseToSnakeCase(label: string): string {