diff --git a/.buildkite/pipeline.graphql.yml b/.buildkite/pipeline.graphql.yml
index 1d34fd45ae..d20355fff8 100644
--- a/.buildkite/pipeline.graphql.yml
+++ b/.buildkite/pipeline.graphql.yml
@@ -1,13 +1,3 @@
 steps:
   - label: Update GraphQL docs
     command: .buildkite/update_graphql_docs
-    plugins:
-      - docker-compose#v4.9.0:
-          run: app
-          mount-ssh-agent: true
-          mount-buildkite-agent: true
-          env:
-            - API_ACCESS_TOKEN
-            - GIT_NAME
-            - GIT_EMAIL
-            - GH_TOKEN
diff --git a/.buildkite/update_graphql_docs b/.buildkite/update_graphql_docs
index 29b659c926..a1314fff7d 100755
--- a/.buildkite/update_graphql_docs
+++ b/.buildkite/update_graphql_docs
@@ -4,7 +4,11 @@ set -euo pipefail
 . bin/utils.sh
 
 echo "+++ Fetch latest schema (via GraphQL introspection)"
-rake graphql:fetch_schema >| data/graphql/schema.graphql
+docker compose run \
+  --no-deps \
+  -e API_ACCESS_TOKEN \
+  app \
+  bash -c "rake graphql:fetch_schema >| data/graphql/schema.graphql"
 
 echo "+++ Check for schema changes"
 if git diff --exit-code --quiet data/graphql/schema.graphql; then
@@ -21,12 +25,12 @@ BRANCH=buildkite-docs-bot/graphql/$(git rev-parse --short :data/graphql/schema.g
 PULL_REQUEST_NUMBER=$(get_branch_pull_request_number $BRANCH)
 
 if [ -n "$PULL_REQUEST_NUMBER" ]; then
-  buildkite-agent annotate  "Nothing to change, pull request [#$PULL_REQUEST_NUMBER](https://github.com/buildkite/docs/pull/$PULL_REQUEST_NUMBER) awaiting approval." --style "info"
+  buildkite-agent annotate "Nothing to change, pull request [#$PULL_REQUEST_NUMBER](https://github.com/buildkite/docs/pull/$PULL_REQUEST_NUMBER) awaiting approval." --style "info"
   exit 0
 fi
 
 echo "+++ Generate GraphQL docs"
-./scripts/generate-graphql-api-content.sh
+docker compose run --no-deps app rake graphql:generate
 
 echo "--- Commit and push changes"
 git add .
@@ -37,10 +41,5 @@ git commit -m "Update GraphQL docs"
 git push -u origin $BRANCH
 
 echo "+++ Create pull request"
-gh pr create \
-  --title "Update GraphQL docs" \
-  --body "This is an automated PR based on the current GraphQL schema" \
-  | buildkite-agent annotate --style "success"
-
-# Auto-merge PR once checks have passed
-gh pr merge -m --auto
+create_pull_request "Update GraphQL docs" "This is an automated PR based on the current GraphQL schema" \
+  | jq ".url" | buildkite-agent annotate --style "success"
diff --git a/README.md b/README.md
index 90bfbb386f..3afb8e8ed1 100644
--- a/README.md
+++ b/README.md
@@ -81,7 +81,7 @@ If you need to fetch the latest schema you can either:
 API_ACCESS_TOKEN=xxx rake graphql:fetch_schema >| data/graphql/schema.graphql
 
 # Generate docs based on latest schema
-./scripts/generate-graphql-api-content.sh
+rake graphql:generate
 ```
 
 
diff --git a/bin/utils.sh b/bin/utils.sh
index 8ff1c858b1..22753756c9 100644
--- a/bin/utils.sh
+++ b/bin/utils.sh
@@ -33,3 +33,13 @@ function post_github_comment() {
     --data "{\"body\":\"$2\"}" \
     https://api.github.com/repos/buildkite/docs/issues/$1/comments
 }
+
+function create_pull_request() {
+  curl -L \
+    -X POST \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $GH_TOKEN" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    --data "{\"title\":\"$1\", "body\":\"$2\"}" \
+    https://api.github.com/repos/buildkite/docs/pulls"
+}
diff --git a/lib/tasks/graphql.rake b/lib/tasks/graphql.rake
index 89e81d109a..76309e9e4c 100644
--- a/lib/tasks/graphql.rake
+++ b/lib/tasks/graphql.rake
@@ -16,4 +16,9 @@ namespace :graphql do
 
     puts GraphQL::Client.load_schema(HTTP).to_definition
   end
+
+  desc "Generate GraphQL docs and navigation from GraphQL schema"
+  task :generate do
+    ruby "scripts/generate_graphql_api_content.rb"
+  end
 end
diff --git a/scripts/generate-graphql-api-content.sh b/scripts/generate-graphql-api-content.sh
deleted file mode 100755
index 355b8523bd..0000000000
--- a/scripts/generate-graphql-api-content.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-scripts_dir="$(dirname "${BASH_SOURCE[0]}")"
-
-rm -rf ${scripts_dir}/../pages/apis/graphql/schemas/*
-ruby "${scripts_dir}/generate_graphql_api_content.rb"
diff --git a/scripts/generate_graphql_api_content.rb b/scripts/generate_graphql_api_content.rb
index 6d65611959..3162ae7c89 100644
--- a/scripts/generate_graphql_api_content.rb
+++ b/scripts/generate_graphql_api_content.rb
@@ -33,7 +33,7 @@
         schema_type_data["kind"].to_s.downcase
       end
 
-      Dir.mkdir("#{schemas_dir}/#{sub_dir}") unless File.exists?("#{schemas_dir}/#{sub_dir}")
+      Dir.mkdir("#{schemas_dir}/#{sub_dir}") unless File.exist?("#{schemas_dir}/#{sub_dir}")
       File.write("#{schemas_dir}/#{sub_dir}/#{name.downcase}.md", render_page(schema_type_data, sub_dir.capitalize.pluralize))
     end
   end