diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index fa49025d..9e484516 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     container: quay.io/cortexproject/cortex-jsonnet-build-image:e158eda
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         name: Checkout
         with:
           fetch-depth: 0
@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     container: quay.io/cortexproject/cortex-jsonnet-build-image:e158eda
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         name: Checkout
         with:
           fetch-depth: 0
@@ -36,10 +36,14 @@ jobs:
     runs-on: ubuntu-latest
     container: quay.io/cortexproject/cortex-jsonnet-build-image:e158eda
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         name: Checkout
         with:
           fetch-depth: 0
       
-      - name: "Test readme"
-        run: make test-readme
+      - name: "Test readme s3"
+        run: make test-readme/s3
+      - name: "Test readme azure"
+        run: make test-readme/azure
+      - name: "Test readme gcs"
+        run: make test-readme/gcs
diff --git a/Makefile b/Makefile
index 3b1f77d1..7ddbd645 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: lint build-image publish-build-image test-readme
+.PHONY: lint build-image publish-build-image test-readme clean
 
 JSONNET_FMT := jsonnetfmt
 
@@ -47,14 +47,7 @@ build-mixin:
 test-readme: test-readme/azure test-readme/gcs test-readme/s3
 
 test-readme/%:
-	rm -rf $@ && \
-	mkdir -p $@ && cd $@ && \
-	tk init --k8s=1.24 && \
-	jb install github.com/cortexproject/cortex-jsonnet/cortex@main && \
-	rm -fr ./vendor/cortex && \
-	cp -r ../../cortex ./vendor/ && \
-	cp vendor/cortex/$(notdir $@)/main.jsonnet.example environments/default/main.jsonnet && \
-	PAGER=cat tk show environments/default
+	@./scripts/test-readme.sh $@
 
 clean-white-noise:
 	@$(FIND) . -type f -regextype posix-extended -regex '.*(md|libsonnet)' -print | \
@@ -62,3 +55,9 @@ clean-white-noise:
 
 check-white-noise: clean-white-noise
 	@git diff --exit-code --quiet || (echo "Please remove trailing whitespaces running 'make clean-white-noise'" && false)
+
+clean:
+	rm -rf cortex-mixin/out
+	rm -rf cortex-mixin/vendor
+	rm -f cortex-mixin/cortex-mixin.zip
+	rm -rf test-readme
diff --git a/scripts/test-readme.sh b/scripts/test-readme.sh
new file mode 100755
index 00000000..84a51a60
--- /dev/null
+++ b/scripts/test-readme.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+set -xe
+rm -rf $1
+mkdir -p $1
+cd $1
+tk init --k8s=1.26
+jb install github.com/cortexproject/cortex-jsonnet/cortex@main
+rm -fr ./vendor/cortex
+cp -r ../../cortex ./vendor/
+cp vendor/cortex/$(basename $1)/main.jsonnet.example environments/default/main.jsonnet
+PAGER=cat tk show --dangerous-allow-redirect environments/default