diff --git a/.github/workflows/build_test_containers.yml b/.github/workflows/build_test_containers.yml
index b160110022..eb66ce3f47 100644
--- a/.github/workflows/build_test_containers.yml
+++ b/.github/workflows/build_test_containers.yml
@@ -35,14 +35,15 @@ jobs:
               id: get_changed_files
               run: |
                 if [ ${{ github.event_name }} == "push" ]; then
-                    changed_files=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep ansible/docker/test/Dockerfile)
+                    changed_files=$(git diff --name-only HEAD~1 | grep -v ansible/docker/test/Dockerfile)
                 else
-                    changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | grep ansible/docker/test/Dockerfile)
+                    changed_files=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | grep -v ansible/docker/test/Dockerfile)
                 fi
                 echo "changed_files=$changed_files" >> "$GITHUB_ENV"
 
             # Generate matrix
             - name: Generate matrix
+              if: steps.get_changed_files.outputs.changed_files
               id: generate_matrix
               run: |
                 matrix=$(jq -n --arg files "${changed_files}" '{