Move processes to reusable actions (#1266)

Moved the SSH setup to a reusable action --------- Co-authored-by: STEFANO BOSISIO <[email protected]>
NVIDIA · Jan 28, 2025 · 45d5750 · 45d5750
1 parent 1a13844
commit 45d5750
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 127 deletions.
diff --git a/.github/actions/setup-ssh/action.yml b/.github/actions/setup-ssh/action.yml
@@ -0,0 +1,32 @@
+name: 'Setup SSH'
+
+description: 'Setup SSH agent and known hosts'
+
+inputs:
+  ssh-private-key:
+    description: 'SSH private key'
+    required: true
+  ssh-known-hosts:
+    description: 'SSH Known Hosts entries'
+    required: true
+outputs:
+  known-hosts-file:
+    description: 'Path to the SSH known hosts file'
+    value: ${{ steps.ssh-known-hosts.outputs.known_hosts_file }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup SSH agent
+      uses: webfactory/[email protected]
+      with:
+        ssh-private-key: ${{ inputs.ssh-private-key }}
+
+    - name: Setup SSH known hosts
+      id: ssh-known-hosts
+      shell: bash
+      run: |
+        mkdir -p ~/.ssh
+        echo "${{ inputs.ssh-known-hosts }}" >> ~/.ssh/known_hosts
+        chmod 600 ~/.ssh/known_hosts
+        echo "known_hosts_file=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml
@@ -77,24 +77,16 @@ jobs:
       - name: Print environment variables
         run: env
 
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
-
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -134,7 +126,7 @@ jobs:
           labels: ${{ steps.mealkit-metadata.outputs.labels }}
           ssh: default
           secret-files: |
-            "SSH_KNOWN_HOSTS=${{ steps.ssh-known-hosts.outputs.FILE }}"
+            "SSH_KNOWN_HOSTS=${{ steps.setup-ssh.outputs.known-hosts-file }}"
           build-args: |
             BASE_IMAGE=${{ inputs.BASE_IMAGE }}
             BAZEL_CACHE=${{ vars.BAZEL_REMOTE_CACHE_URL }}
@@ -167,7 +159,7 @@ jobs:
           target: final
           ssh: default
           secret-files: |
-            "SSH_KNOWN_HOSTS=${{ steps.ssh-known-hosts.outputs.FILE }}"
+            "SSH_KNOWN_HOSTS=${{ steps.setup-ssh.outputs.known-hosts-file }}"
           build-args: |
             BASE_IMAGE=${{ inputs.BASE_IMAGE }}
             BAZEL_CACHE=${{ vars.BAZEL_REMOTE_CACHE_URL }}

diff --git a/.github/workflows/_publish_t5x_pax_results.yaml b/.github/workflows/_publish_t5x_pax_results.yaml
@@ -22,29 +22,12 @@ jobs:
   publish:
     runs-on: ubuntu-22.04
     steps:
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
-
-      - name: Setup SSH config
-        id: ssh-config
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/config << EOF
-          ${{ vars.SSH_CONFIG }}
-          EOF
-          chmod 600 ~/.ssh/config
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
 
       - name: Create dated folder and generate TensorBoard query URL
         id: mkdir

diff --git a/.github/workflows/_runner_ondemand_slurm.yaml b/.github/workflows/_runner_ondemand_slurm.yaml
@@ -28,20 +28,12 @@ jobs:
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
 
       - name: Labels and metadata
         id: meta

diff --git a/.github/workflows/_test_maxtext.yaml b/.github/workflows/_test_maxtext.yaml
@@ -51,20 +51,12 @@ jobs:
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
 
       - name: Labels and metadata
         id: meta

diff --git a/.github/workflows/_test_pax_rosetta.yaml b/.github/workflows/_test_pax_rosetta.yaml
@@ -45,23 +45,15 @@ jobs:
       - name: Print environment variables
         run: env
 
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
 
       - name: Labels and metadata
         id: meta

diff --git a/.github/workflows/_test_slurm_pyxis.yaml b/.github/workflows/_test_slurm_pyxis.yaml
@@ -93,21 +93,13 @@ jobs:
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
-
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
+
       - name: Labels and metadata
         id: meta
         shell: bash -x -e {0}

diff --git a/.github/workflows/_test_t5x_rosetta.yaml b/.github/workflows/_test_t5x_rosetta.yaml
@@ -54,24 +54,16 @@ jobs:
     steps:
       - name: Print environment variables
         run: env
-
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
 
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
+        with:
+          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
 
       - name: Labels and metadata
         id: meta

diff --git a/.github/workflows/_test_upstream_pax.yaml b/.github/workflows/_test_upstream_pax.yaml
@@ -46,20 +46,12 @@ jobs:
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
 
       - name: Labels and metadata
         id: meta

diff --git a/.github/workflows/_test_upstream_t5x.yaml b/.github/workflows/_test_upstream_t5x.yaml
@@ -54,20 +54,12 @@ jobs:
       - name: Check out the repository under ${GITHUB_WORKSPACE}
         uses: actions/checkout@v4
 
-      - name: Setup SSH agent
-        uses: webfactory/[email protected]
+      - name: Setup SSH
+        id: setup-ssh
+        uses: ./.github/actions/setup-ssh
         with:
           ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Setup SSH known hosts
-        id: ssh-known-hosts
-        run: |
-          mkdir -p ~/.ssh
-          cat >> ~/.ssh/known_hosts << EOF
-          ${{ vars.SSH_KNOWN_HOSTS }}
-          EOF
-          chmod 600 ~/.ssh/known_hosts
-          echo "FILE=$(realpath ~/.ssh/known_hosts)" >> $GITHUB_OUTPUT
+          ssh-known-hosts: ${{ vars.SSH_KNOWN_HOSTS }}
 
       - name: Labels and metadata
         id: meta