File cache workload

perfmetrics/scripts/testing_on_gke/examples/parallel_downloads/10g_file_cache.yaml

@@ -0,0 +1,69 @@
+{
+  "_comment": "_ in the starting of element name indicates comment.",
+  "TestConfig": {
+    "workloadConfig": {
+      "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.",
+      "runOnSSD": false,
+      "workloads": [
+        {
+          "_comment": "File-cache enabled"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 16,
+            "numThreads": 1,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 2,
+            "numThreads": 8,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 2,
+            "numThreads": 16,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 1,
+            "numThreads": 64,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 1,
+            "numThreads": 96,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-10g-500-us-west1"
+        }
+      ]
+    }
+  }
+}

perfmetrics/scripts/testing_on_gke/examples/parallel_downloads/10g_normal.yaml

@@ -0,0 +1,69 @@
+{
+  "_comment": "_ in the starting of element name indicates comment.",
+  "TestConfig": {
+    "workloadConfig": {
+      "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.",
+      "runOnSSD": false,
+      "workloads": [
+        {
+          "_comment": "file cache disabled"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 16,
+            "numThreads": 1,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 2,
+            "numThreads": 8,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 2,
+            "numThreads": 16,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 1,
+            "numThreads": 64,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 1,
+            "numThreads": 96,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-10g-500-us-west1"
+        }
+      ]
+    }
+  }
+}

perfmetrics/scripts/testing_on_gke/examples/parallel_downloads/10g_pd.yaml

@@ -0,0 +1,70 @@
+{
+  "_comment": "_ in the starting of element name indicates comment.",
+  "TestConfig": {
+    "workloadConfig": {
+      "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.",
+      "runOnSSD": false,
+      "workloads": [
+
+        {
+          "_comment": "parallel downloads enabled"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 16,
+            "numThreads": 1,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 2,
+            "numThreads": 8,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 2,
+            "numThreads": 16,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 1,
+            "numThreads": 64,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true",
+          "bucket":"fio-10g-500-us-west1"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "10G",
+            "filesPerThread": 1,
+            "numThreads": 96,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true,file-cache:enable-parallel-downloads:true",
+          "bucket":"fio-10g-500-us-west1"
+        }
+      ]
+    }
+  }
+}

perfmetrics/scripts/testing_on_gke/examples/parallel_downloads/all_combo_file_cache.yaml

@@ -0,0 +1,89 @@
+{
+  "_comment": "_ in the starting of element name indicates comment.",
+  "TestConfig": {
+    "workloadConfig": {
+      "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.",
+      "runOnSSD": false,
+      "workloads": [
+        {
+          "_comment": "File-cache enabled"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "128K",
+            "filesPerThread": 20000,
+            "numThreads": 50,
+            "blockSize": "128K",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-128k-1m-us-west1",
+          "_bucket_alt2":"fio-128k-1m-us-central1",
+          "_bucket_alt3":"gke-fio-128k-1m"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "1M",
+            "filesPerThread": 20000,
+            "numThreads": 50,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-1mb-1m-us-west1",
+          "_bucket_alt2":"fio-1mb-1m-us-central1",
+          "_bucket_alt3":"gke-fio-1mb-1m"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "100M",
+            "filesPerThread": 1000,
+            "numThreads": 50,
+            "blockSize": "1M",
+            "readTypes": ["read","randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-100mb-50k-us-west1",
+          "_bucket_alt2":"fio-100mb-50k-us-central1",
+          "_bucket_alt3":"gke-fio-100mb-50k"
+        },
+        {
+          "fioWorkload": {
+            "_description": "This workload's job file is configured differently from the rest. It has one file, whis is read in parallel depending on the value of numThreads (only 100 supported right now).",
+            "fileSize": "200G",
+            "filesPerThread": 1,
+            "numThreads": 100,
+            "blockSize": "1M",
+            "readTypes": ["read","randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"fio-200gb-1-us-west1",
+          "_bucket_alt2":"fio-200gb-1-us-central1",
+          "_bucket_alt3":"gke-fio-200gb-1"
+        },
+        {
+          "dlioWorkload": {
+            "numFilesTrain": 500000,
+            "recordLength": 102400,
+            "batchSizes": [128]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"dlio-unet3d-100kb-500k-us-west1",
+          "_bucket_alt2":"dlio-unet3d-100kb-500k-us-central1",
+          "_bucket_alt3":"gke-dlio-unet3d-100kb-500k"
+        },
+        {
+          "dlioWorkload": {
+            "numFilesTrain": 5000,
+            "recordLength": 157286400,
+            "batchSizes": [4]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1,file-cache:max-size-mb:-1,file-cache:cache-file-for-range-read:true",
+          "bucket":"dlio-unet3d-150mb-5k-us-west1",
+          "_bucket_alt2":"dlio-unet3d-150mb-5k-us-central1",
+          "_bucket_alt3":"gke-dlio-unet3d-150mb-5k"
+        }
+      ]
+    }
+  }
+}

perfmetrics/scripts/testing_on_gke/examples/parallel_downloads/all_combo_normal.yaml

@@ -0,0 +1,90 @@
+{
+  "_comment": "_ in the starting of element name indicates comment.",
+  "TestConfig": {
+    "workloadConfig": {
+      "_description": "workloadConfig has an optional field runOnSSD (default true if missing), and an array of workloads.",
+      "runOnSSD": false,
+      "workloads": [
+        {
+          "_comment": "file cache disabled"
+        },
+
+        {
+          "fioWorkload": {
+            "fileSize": "128K",
+            "filesPerThread": 20000,
+            "numThreads": 50,
+            "blockSize": "128K",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-128k-1m-us-west1",
+          "_bucket_alt2":"fio-128k-1m-us-central1",
+          "_bucket_alt3":"gke-fio-128k-1m"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "1M",
+            "filesPerThread": 20000,
+            "numThreads": 50,
+            "blockSize": "1M",
+            "readTypes": ["read"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-1mb-1m-us-west1",
+          "_bucket_alt2":"fio-1mb-1m-us-central1",
+          "_bucket_alt3":"gke-fio-1mb-1m"
+        },
+        {
+          "fioWorkload": {
+            "fileSize": "100M",
+            "filesPerThread": 1000,
+            "numThreads": 50,
+            "blockSize": "1M",
+            "readTypes": ["read", "randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-100mb-50k-us-west1",
+          "_bucket_alt2":"fio-100mb-50k-us-central1",
+          "_bucket_alt3":"gke-fio-100mb-50k"
+        },
+        {
+          "fioWorkload": {
+            "_description": "This workload's job file is configured differently from the rest. It has one file, whis is read in parallel depending on the value of numThreads (only 100 supported right now).",
+            "fileSize": "200G",
+            "filesPerThread": 1,
+            "numThreads": 100,
+            "blockSize": "1M",
+            "readTypes": ["read","randread"]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"fio-200gb-1-us-west1",
+          "_bucket_alt2":"fio-200gb-1-us-central1",
+          "_bucket_alt3":"gke-fio-200gb-1"
+        },
+        {
+          "dlioWorkload": {
+            "numFilesTrain": 500000,
+            "recordLength": 102400,
+            "batchSizes": [128]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"dlio-unet3d-100kb-500k-us-west1",
+          "_bucket_alt2":"dlio-unet3d-100kb-500k-us-central1",
+          "_bucket_alt3":"gke-dlio-unet3d-100kb-500k"
+        },
+        {
+          "dlioWorkload": {
+            "numFilesTrain": 5000,
+            "recordLength": 157286400,
+            "batchSizes": [4]
+          },
+          "gcsfuseMountOptions": "implicit-dirs,metadata-cache:ttl-secs:-1,metadata-cache:type-cache-max-size-mb:-1,metadata-cache:stat-cache-max-size-mb:-1",
+          "bucket":"dlio-unet3d-150mb-5k-us-west1",
+          "_bucket_alt2":"dlio-unet3d-150mb-5k-us-central1",
+          "_bucket_alt3":"gke-dlio-unet3d-150mb-5k"
+        }
+      ]
+    }
+  }
+}