Run unify projection step in parallel

batch/scripts/_warp_and_upload.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# arguments:
+# $0 - The name of this script
+# $1 - remote_src_file
+# $2 - local_src_file
+# $3 - local_warped_file
+# $4 - target_crs
+# $5 - remote target file
+
+if aws s3 ls "$1"; then
+  echo "Remote target file $1 already exists, skipping..."
+  exit 0
+fi
+
+echo "Now downloading $1 to $2"
+time gsutil cp "$1" "$2"
+echo "Done downloading $1 to $2"
+
+echo "Now warping $2 to $3"
+time gdalwarp "$2" "$3" -t_srs "$4" -co COMPRESS=DEFLATE -co TILED=yes
+echo "Done warping $2 to $3"
+
+echo "Now uploading $3 to $5"
+time aws s3 cp "$3" "$5"
+echo "Done uploading $3 to $5"
+
+echo "Finally, deleting local files $2 and $3"
+rm "$2" "$3"
+echo "Done deleting local files $2 and $3"

batch/scripts/unify_projection.sh

@@ -19,32 +19,19 @@ files="02.tif 03.tif 04.tif 05.tif 06.tif 07.tif 08.tif 09.tif 10.tif 11.tif 12.
 
 src_count=0
 
+CMD_ARGS=()
+
 cd /tmp
 for s in ${SRC}; do
   for f in ${files}; do
-    remote_target_file=${TARGET}/SRC_${src_count}/${f}
-    if aws s3 ls ${remote_target_file}; then
-      echo "Remote target file ${remote_target_file} already exists, skipping..."
-      continue
-    fi
-
     remote_src_file=${s}/${f}
     local_src_file=SRC_${src_count}/${f}
-    echo "Now downloading ${remote_src_file} to ${local_src_file}"
-    time gsutil cp ${s}/${f} ${local_src_file}
-    echo "Done"
-
     local_warped_file=REPROJECTED_${src_count}/${f}
-    echo "Now warping ${local_src_file} to ${local_warped_file}"
-    time gdalwarp ${local_src_file} ${local_warped_file} -t_srs "${TARGET_CRS}" -co COMPRESS=DEFLATE -co TILED=yes
-    echo "Done warping ${local_src_file} to ${local_warped_file}"
-
-    echo "Now uploading ${local_warped_file} to ${remote_target_file}"
-    time aws s3 cp ${local_warped_file} ${remote_target_file}
-    echo "Done uploading ${local_warped_file} to ${remote_target_file}"
+    remote_target_file=${TARGET}/SRC_${src_count}/${f}
 
-    echo "Finally, deleting local files ${local_src_file} and ${local_warped_file}"
-    rm ${local_src_file} ${local_warped_file}
+    CMD_ARGS+=("${remote_src_file}" "$local_src_file" "$local_warped_file" "$TARGET_CRS" "$remote_target_file")
   done
   ((count++))
-done
+done
+
+echo $CMD_ARGS | xargs -n 5 -P 32 _warp_and_upload.sh