Skip to content

Commit

Permalink
Merge pull request #20 from ctuning/main
Browse files Browse the repository at this point in the history 
Added derived power efficiency metric
  • Loading branch information
@arjunsuresh
arjunsuresh authored Sep 12, 2023
2 parents 390e590 + 5a4d5ad commit 425d689
Show file tree
Hide file tree
Showing 182 changed files with 40,548 additions and 17,128 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ We created a [sample CM script](script/process-mlperf-inference-results) in this
that you can use and [extend](script/process-mlperf-inference-results/customize.py) to add derived metrics:

```bash
cm run script "process mlperf-inference results" --experiment_tags=mlperf-inference,v3.0
cm run script "process mlperf-inference results" --experiment_tags=mlperf-inference,v3.1
```


Expand Down
4 changes: 3 additions & 1 deletion ...ent/mlperf-inference--all--datacenter--closed-network--object-detection--offline/_cm.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
"datacenter",
"closed-network",
"object-detection",
"offline"
"offline",
"inference",
"network"
],
"uid": "b013db23ca6648ca"
}
4 changes: 3 additions & 1 deletion ...ment/mlperf-inference--all--datacenter--closed-network--object-detection--server/_cm.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
"datacenter",
"closed-network",
"object-detection",
"server"
"server",
"inference",
"network"
],
"uid": "82bf74dbd7604d93"
}
66 changes: 50 additions & 16 deletions ...er--closed-power--image-classification--offline/2023-04-06.00-36-37.490606/cm-result.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@
"Accuracy_div_100": 0.76076,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 153.6838256159247,
"Location": "closed/NVIDIA/results/H100-PCIe-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "NVIDIA",
"Platform": "H100-PCIe-80GBx8_TRT_MaxQ",
"Result": 348572,
"Result_Power": 2268.1111600587396,
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "Gigabyte G482-Z54 (8x H100-PCIe-80GB, MaxQ, TensorRT)",
Expand Down Expand Up @@ -38,18 +41,20 @@
"Accuracy_div_100": 0.76076,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 153.6838256159247,
"Location": "closed/NVIDIA/results/H100-PCIe-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "NVIDIA",
"Platform": "H100-PCIe-80GBx8_TRT_MaxQ",
"Result": null,
"Result": 348572,
"Result_Power": 2268.1111600587396,
"Result_Units": "Watts",
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "Gigabyte G482-Z54 (8x H100-PCIe-80GB, MaxQ, TensorRT)",
"SystemType": "datacenter",
"Units": "Watts",
"Units": "Samples/s",
"accelerator_model_name": "NVIDIA H100-PCIe-80GB",
"accelerators_per_node": 8,
"compliance": 1,
Expand All @@ -64,7 +69,7 @@
"notes": "",
"number_of_nodes": 1,
"operating_system": "Ubuntu 20.04.4",
"uid": "c72597c4ddb54385",
"uid": "43cd3b0c542043d2",
"url": "https://github.com/mlcommons/inference_results_v3.1/tree/master/closed/NVIDIA/results/H100-PCIe-80GBx8_TRT_MaxQ/resnet50/Offline",
"version": "v3.1"
},
Expand All @@ -73,12 +78,15 @@
"Accuracy_div_100": 0.76076,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 116.84324852007448,
"Location": "closed/NVIDIA/results/DGX-H100_H100-SXM-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "NVIDIA",
"Platform": "DGX-H100_H100-SXM-80GBx8_TRT_MaxQ",
"Result": 474849,
"Result_Power": 4063.983208395799,
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "NVIDIA DGX H100 (8x H100-SXM-80GB, MaxQ, TensorRT)",
Expand Down Expand Up @@ -107,18 +115,20 @@
"Accuracy_div_100": 0.76076,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 116.84324852007448,
"Location": "closed/NVIDIA/results/DGX-H100_H100-SXM-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "NVIDIA",
"Platform": "DGX-H100_H100-SXM-80GBx8_TRT_MaxQ",
"Result": null,
"Result": 474849,
"Result_Power": 4063.983208395799,
"Result_Units": "Watts",
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "NVIDIA DGX H100 (8x H100-SXM-80GB, MaxQ, TensorRT)",
"SystemType": "datacenter",
"Units": "Watts",
"Units": "Samples/s",
"accelerator_model_name": "NVIDIA H100-SXM-80GB",
"accelerators_per_node": 8,
"compliance": 1,
Expand All @@ -133,7 +143,7 @@
"notes": "",
"number_of_nodes": 1,
"operating_system": "Ubuntu 20.04.4",
"uid": "1bc1af8ef3f64ea1",
"uid": "38c78184a53f41a8",
"url": "https://github.com/mlcommons/inference_results_v3.1/tree/master/closed/NVIDIA/results/DGX-H100_H100-SXM-80GBx8_TRT_MaxQ/resnet50/Offline",
"version": "v3.1"
},
Expand All @@ -142,12 +152,15 @@
"Accuracy_div_100": 0.75936,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 236.95563565124138,
"Location": "closed/Qualcomm/results/g292_z43_q16e/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "Qualcomm",
"Platform": "g292_z43_q16e",
"Result": 337737,
"Result_Power": 1425.3174399999996,
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "GIGABYTE G292-Z43 (16x QAIC100 Pro, EE)",
Expand Down Expand Up @@ -176,18 +189,20 @@
"Accuracy_div_100": 0.75936,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 236.95563565124138,
"Location": "closed/Qualcomm/results/g292_z43_q16e/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "Qualcomm",
"Platform": "g292_z43_q16e",
"Result": null,
"Result": 337737,
"Result_Power": 1425.3174399999996,
"Result_Units": "Watts",
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "GIGABYTE G292-Z43 (16x QAIC100 Pro, EE)",
"SystemType": "datacenter",
"Units": "Watts",
"Units": "Samples/s",
"accelerator_model_name": "QUALCOMM Cloud AI 100 PCIe/HHHL Pro",
"accelerators_per_node": 16,
"compliance": 1,
Expand All @@ -202,7 +217,7 @@
"notes": "With 75W Accelerator TDP constraints. Powered by the KRAI X and KILT technologies",
"number_of_nodes": 1,
"operating_system": "Ubuntu 20.04.6 LTS (Linux kernel 5.4.0-139-generic #156-Ubuntu SMP Fri Jan 20 17:27:18 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux)",
"uid": "24024aeca40142ba",
"uid": "d986eb16ba814c7d",
"url": "https://github.com/mlcommons/inference_results_v3.1/tree/master/closed/Qualcomm/results/g292_z43_q16e/resnet50/offline",
"version": "v3.1"
},
Expand All @@ -211,12 +226,15 @@
"Accuracy_div_100": 0.75936,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 247.72738273950966,
"Location": "closed/Qualcomm/results/r282_z93_q8e/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "Qualcomm",
"Platform": "r282_z93_q8e",
"Result": 169969,
"Result_Power": 686.1130898021307,
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "GIGABYTE R282-Z93 (8x QAIC100 Pro, EE)",
Expand Down Expand Up @@ -245,18 +263,20 @@
"Accuracy_div_100": 0.75936,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 247.72738273950966,
"Location": "closed/Qualcomm/results/r282_z93_q8e/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
"Organization": "Qualcomm",
"Platform": "r282_z93_q8e",
"Result": null,
"Result": 169969,
"Result_Power": 686.1130898021307,
"Result_Units": "Watts",
"Result_Power_Units": "Watts",
"Result_Units": "Samples/s",
"Scenario": "Offline",
"SystemName": "GIGABYTE R282-Z93 (8x QAIC100 Pro, EE)",
"SystemType": "datacenter",
"Units": "Watts",
"Units": "Samples/s",
"accelerator_model_name": "QUALCOMM Cloud AI 100 PCIe/HHHL Pro",
"accelerators_per_node": 8,
"compliance": 1,
Expand All @@ -271,7 +291,7 @@
"notes": "With 75W Accelerator TDP constraints. 3x QAIC100 on riser CRS2033; 3x QAIC100 on riser CRS2033; 2x QAIC100 on riser CRS2026. Powered by the KRAI X and KILT technologies",
"number_of_nodes": 1,
"operating_system": "Ubuntu 20.04.6 LTS (Linux kernel: 5.4.0-139-generic #156-Ubuntu SMP Fri Jan 20 17:27:18 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux)",
"uid": "4f65f4ad23c54089",
"uid": "b8ecc6ec17fb44f6",
"url": "https://github.com/mlcommons/inference_results_v3.1/tree/master/closed/Qualcomm/results/r282_z93_q8e/resnet50/offline",
"version": "v3.1"
},
Expand All @@ -280,6 +300,7 @@
"Accuracy_div_100": 0.75956,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 207.34882071229413,
"Location": "closed/Krai/results/g292_z43_q18e-qaic-v1.7.1.12-aic100/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -316,6 +337,7 @@
"Accuracy_div_100": 0.76072,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 122.18765136775356,
"Location": "closed/NVIDIA/results/A100-PCIe-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -352,6 +374,7 @@
"Accuracy_div_100": 0.76072,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 93.67548704914302,
"Location": "closed/NVIDIA/results/DGX-A100_A100-SXM-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -388,6 +411,7 @@
"Accuracy_div_100": 0.75956,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 216.9987377836515,
"Location": "closed/Qualcomm/results/r282_z93_q8e-qaic-v1.7.1.12-aic100/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -424,6 +448,7 @@
"Accuracy_div_100": 0.76072,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 11.582589520406332,
"Location": "closed/Dell/results/XR12_A2x1_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -460,6 +485,7 @@
"Accuracy_div_100": 0.75956,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 183.67951208453943,
"Location": "closed/Dell/results/r7515_q4_pro-qaic-v1.7.1.12-aic100/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -496,6 +522,7 @@
"Accuracy_div_100": 0.76072,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 94.43937912972616,
"Location": "closed/Dell/results/XE8545_A100_SXM_80GBx4_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -532,6 +559,7 @@
"Accuracy_div_100": 0.76072,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 63.84678980811712,
"Location": "closed/H3C/results/A30x3_R4900G5_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -568,6 +596,7 @@
"Accuracy_div_100": 0.76,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 124.93499333081373,
"Location": "closed/NVIDIA/results/A100-PCIe-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -604,6 +633,7 @@
"Accuracy_div_100": 0.76156,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 159.14380714211697,
"Location": "closed/NVIDIA/results/H100-PCIe-80GBx8_TRT_MaxQ/resnet50/Offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -640,6 +670,7 @@
"Accuracy_div_100": 0.7594,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 241.7379091729014,
"Location": "closed/Qualcomm/results/r282_z93_q8e-qaic-v1.8.3.7-aic100/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -676,6 +707,7 @@
"Accuracy_div_100": 0.7594,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 197.5726506236504,
"Location": "closed/Qualcomm/results/g292_z43_q18e-qaic-v1.8.3.7-aic100/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -712,6 +744,7 @@
"Accuracy_div_100": 0.7594,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 210.73579904204945,
"Location": "closed/Qualcomm/results/g292_z43_q16e-qaic-v1.8.3.7-aic100/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down Expand Up @@ -748,6 +781,7 @@
"Accuracy_div_100": 0.7594,
"Availability": "available",
"Division": "closed",
"Inference_per_Joule": 184.02037557741292,
"Location": "closed/Dell/results/r7515_q4_pro-qaic-v1.8.3.7-aic100/resnet50/offline",
"MlperfModel": "resnet",
"Model": "resnet50",
Expand Down
Loading

0 comments on commit 425d689

Please sign in to comment.