Change dataset download scripts to use Cloudflare buckets directly in…

…stead of going through Nextcloud (#712)
mlcommons · Mar 12, 2024 · 68f8f38 · 68f8f38
1 parent e237206
commit 68f8f38
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 6 deletions.
diff --git a/stable_diffusion/scripts/datasets/coco2014-validation-download-prompts.sh b/stable_diffusion/scripts/datasets/coco2014-validation-download-prompts.sh
@@ -12,4 +12,8 @@ while [ "$1" != "" ]; do
 done
 
 mkdir -p ${OUTPUT_DIR}
-wget -O ${OUTPUT_DIR}/val2014_30k.tsv -c "https://cloud.mlcommons.org/index.php/s/training_stable_diffusion/download?path=/datasets/coco2014&files=val2014_30k.tsv"
+
+rclone config create mlc-training s3 provider=Cloudflare access_key_id=76ea42eadb867e854061a1806220ee1e secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
+
+rclone copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/coco2014/val2014_30k.tsv ${OUTPUT_DIR} -P
+
diff --git a/stable_diffusion/scripts/datasets/coco2014-validation-download-stats.sh b/stable_diffusion/scripts/datasets/coco2014-validation-download-stats.sh
@@ -12,4 +12,8 @@ while [ "$1" != "" ]; do
 done
 
 mkdir -p ${OUTPUT_DIR}
-wget -O ${OUTPUT_DIR}/val2014_30k_stats.npz -c "https://cloud.mlcommons.org/index.php/s/training_stable_diffusion/download?path=/datasets/coco2014&files=val2014_30k_stats.npz"
+
+rclone config create mlc-training s3 provider=Cloudflare access_key_id=76ea42eadb867e854061a1806220ee1e secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
+
+rclone copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/coco2014/val2014_30k_stats.npz ${OUTPUT_DIR} -P
+
diff --git a/stable_diffusion/scripts/datasets/laion400m-filtered-download-images.sh b/stable_diffusion/scripts/datasets/laion400m-filtered-download-images.sh
@@ -15,8 +15,10 @@ mkdir -p ${OUTPUT_DIR}
 cd ${OUTPUT_DIR}
 
 
-for i in {00000..00831}; do wget -O ${OUTPUT_DIR}/${i}.tar -c "https://cloud.mlcommons.org/index.php/s/training_stable_diffusion/download?path=/datasets/laion-400m/images-webdataset-filtered&files=${i}.tar"; done
+rclone config create mlc-training s3 provider=Cloudflare access_key_id=76ea42eadb867e854061a1806220ee1e secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
 
-wget -O ${OUTPUT_DIR}/sha512sums.txt -c "https://cloud.mlcommons.org/index.php/s/training_stable_diffusion/download?path=/datasets/laion-400m/images-webdataset-filtered&files=sha512sums.txt"
+rclone copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/laion-400m/images-webdataset-filtered/ ${OUTPUT_DIR} --include="*.tar" -P
+
+rclone copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/laion-400m/images-webdataset-filtered/sha512sums.txt ${OUTPUT_DIR} -P
 
 sha512sum --quiet -c sha512sums.txt
diff --git a/stable_diffusion/scripts/datasets/laion400m-filtered-download-moments.sh b/stable_diffusion/scripts/datasets/laion400m-filtered-download-moments.sh
@@ -15,8 +15,10 @@ mkdir -p ${OUTPUT_DIR}
 cd ${OUTPUT_DIR}
 
 
-for i in {00000..00831}; do wget -O ${OUTPUT_DIR}/${i}.tar -c "https://cloud.mlcommons.org/index.php/s/training_stable_diffusion/download?path=/datasets/laion-400m/moments-webdataset-filtered&files=${i}.tar"; done
+rclone config create mlc-training s3 provider=Cloudflare access_key_id=76ea42eadb867e854061a1806220ee1e secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
 
-wget -O ${OUTPUT_DIR}/sha512sums.txt -c "https://cloud.mlcommons.org/index.php/s/training_stable_diffusion/download?path=/datasets/laion-400m/moments-webdataset-filtered&files=sha512sums.txt"
+rclone copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/laion-400m/moments-webdataset-filtered/ ${OUTPUT_DIR} --include="*.tar" -P
+
+rclone copy mlc-training:mlcommons-training-wg-public/stable_diffusion/datasets/laion-400m/moments-webdataset-filtered/sha512sums.txt ${OUTPUT_DIR} -P
 
 sha512sum --quiet -c sha512sums.txt