diff --git a/commands/run.sh b/commands/run.sh
index c65ca76..7d6d735 100644
--- a/commands/run.sh
+++ b/commands/run.sh
@@ -348,9 +348,8 @@ fi
 
 if [[ "${BUILDKITE_PLUGIN_DOCKER_ALWAYS_PULL:-false}" =~ ^(true|on|1)$ ]] ; then
   echo "--- :docker: Pulling ${image}"
-  if ! retry "${BUILDKITE_PLUGIN_DOCKER_PULL_RETRIES:-3}" \
-       docker pull "${image}" ; then
-    retry_exit_status="$?"
+  retry "${BUILDKITE_PLUGIN_DOCKER_PULL_RETRIES:-3}" docker pull "${image}" || retry_exit_status="$?"
+  if [ "${retry_exit_status:-0}" -ne 0 ] ; then
     echo "!!! :docker: Pull failed."
     exit "$retry_exit_status"
   fi
diff --git a/lib/shared.bash b/lib/shared.bash
index 1f14f89..807b89a 100644
--- a/lib/shared.bash
+++ b/lib/shared.bash
@@ -8,7 +8,7 @@ function retry {
   local attempts=1
 
   until "$@"; do
-    retry_exit_status=$?
+    local retry_exit_status=$?
     echo "Exited with $retry_exit_status"
     if (( retries == "0" )); then
       return $retry_exit_status
diff --git a/tests/command.bats b/tests/command.bats
index a0112e5..76d64f9 100644
--- a/tests/command.bats
+++ b/tests/command.bats
@@ -66,6 +66,42 @@ setup() {
   unstub docker
 }
 
+@test "Pull image first before running BUILDKITE_COMMAND, success on retries" {
+  export BUILDKITE_PLUGIN_DOCKER_ALWAYS_PULL=true
+  export BUILDKITE_PLUGIN_DOCKER_PULL_RETRIES=2
+
+  stub docker \
+    "pull image:tag" \
+    "pull image:tag : echo pulled latest image on retry" \
+    "run -t -i --rm --init --volume $PWD:/workdir --workdir /workdir --label com.buildkite.job-id=1-2-3-4 image:tag /bin/sh -e -c 'pwd' : echo ran command in docker"
+
+  run "$PWD"/hooks/command
+
+  assert_success
+  assert_output --partial "Retrying 1 more times..."
+  assert_output --partial "pulled latest image on retry"
+  assert_output --partial "ran command in docker"
+
+  unstub docker
+}
+
+@test "Pull image first before running BUILDKITE_COMMAND, failure on retries" {
+  export BUILDKITE_PLUGIN_DOCKER_ALWAYS_PULL=true
+  export BUILDKITE_PLUGIN_DOCKER_PULL_RETRIES=2
+
+  stub docker \
+    "pull image:tag" \
+    "pull image:tag : exit 3"
+
+  run "$PWD"/hooks/command
+
+  assert_failure 3
+  assert_output --partial "Retrying 1 more times..."
+  assert_output --partial "!!! :docker: Pull failed."
+
+  unstub docker
+}
+
 @test "Runs BUILDKITE_COMMAND with mount-buildkite-agent disabled specifically" {
   export BUILDKITE_PLUGIN_DOCKER_MOUNT_BUILDKITE_AGENT=false
   export BUILDKITE_COMMAND="pwd"