From 5935b25622c0483424b2d02883aecefa374ee44f Mon Sep 17 00:00:00 2001 From: Yuwen Hu <54161268+Oscilloscope98@users.noreply.github.com> Date: Fri, 18 Oct 2024 18:15:13 +0800 Subject: [PATCH] Further update windows gpu perf test regarding results integrity check (#12232) --- .github/workflows/llm_performance_tests.yml | 68 +++++++++---------- .../benchmark/igpu-perf/check_csv_results.py | 2 + 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml index f74c91b66da..ba97ed38502 100644 --- a/.github/workflows/llm_performance_tests.yml +++ b/.github/workflows/llm_performance_tests.yml @@ -783,7 +783,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -814,7 +814,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_436.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -845,7 +845,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_438.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) + REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -882,7 +882,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_443.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) + REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -916,7 +916,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_445.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) + REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -971,7 +971,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1002,7 +1002,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_436.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1033,7 +1033,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_438.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1070,7 +1070,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_443.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1104,7 +1104,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_445.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1158,7 +1158,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1189,7 +1189,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_436.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1220,7 +1220,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_438.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1257,7 +1257,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_443.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1291,7 +1291,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_445.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1345,7 +1345,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1379,7 +1379,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_436.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1415,7 +1415,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_438.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) if "${{ matrix.platform }}"=="perf-igpu" ( python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 ) @@ -1462,7 +1462,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_443.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) if "${{ matrix.platform }}"=="perf-igpu" ( python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 ) @@ -1506,7 +1506,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_445.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) if "${{ matrix.platform }}"=="perf-igpu" ( python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 ) @@ -1570,7 +1570,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1601,7 +1601,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_438.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1637,7 +1637,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_443.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1671,7 +1671,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_445.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1727,7 +1727,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1760,7 +1760,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_436.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1793,7 +1793,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_438.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1827,7 +1827,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_443.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1863,7 +1863,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_445.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1919,7 +1919,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1952,7 +1952,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_436.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -1985,7 +1985,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_438.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -2019,7 +2019,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_443.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4 if %ERRORLEVEL% neq 0 (exit /b 1) @@ -2055,7 +2055,7 @@ jobs: move ..\..\..\test\benchmark\igpu-perf\1024-128_445.yaml config.yaml set PYTHONIOENCODING=utf-8 python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + REM if %ERRORLEVEL% neq 0 (exit /b 1) python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5 if %ERRORLEVEL% neq 0 (exit /b 1) diff --git a/python/llm/test/benchmark/igpu-perf/check_csv_results.py b/python/llm/test/benchmark/igpu-perf/check_csv_results.py index a842443d119..ed8a4179603 100644 --- a/python/llm/test/benchmark/igpu-perf/check_csv_results.py +++ b/python/llm/test/benchmark/igpu-perf/check_csv_results.py @@ -65,6 +65,8 @@ def main(): if test_case not in actual_test_cases and test_case not in exclude_test_cases: print(test_case) raise ValueError("The above tests failed. Please check the errors in the log.") + else: + print("Results integrity check completed") if __name__ == "__main__": sys.exit(main())