diff --git a/.buildkite/update_graphql_docs b/.buildkite/update_graphql_docs
index e4688ba6b9..a1314fff7d 100755
--- a/.buildkite/update_graphql_docs
+++ b/.buildkite/update_graphql_docs
@@ -41,7 +41,5 @@ git commit -m "Update GraphQL docs"
 git push -u origin $BRANCH
 
 echo "+++ Create pull request"
-gh pr create \
-  --title "Update GraphQL docs" \
-  --body "This is an automated PR based on the current GraphQL schema" \
-  | buildkite-agent annotate --style "success"
+create_pull_request "Update GraphQL docs" "This is an automated PR based on the current GraphQL schema" \
+  | jq ".url" | buildkite-agent annotate --style "success"
diff --git a/bin/utils.sh b/bin/utils.sh
index 8ff1c858b1..22753756c9 100644
--- a/bin/utils.sh
+++ b/bin/utils.sh
@@ -33,3 +33,13 @@ function post_github_comment() {
     --data "{\"body\":\"$2\"}" \
     https://api.github.com/repos/buildkite/docs/issues/$1/comments
 }
+
+function create_pull_request() {
+  curl -L \
+    -X POST \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $GH_TOKEN" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    --data "{\"title\":\"$1\", "body\":\"$2\"}" \
+    https://api.github.com/repos/buildkite/docs/pulls"
+}