Skip to content

Commit

Permalink
Further update windows gpu perf test regarding results integrity check (
Browse files Browse the repository at this point in the history 
#12232)
  • Loading branch information
@Oscilloscope98
Oscilloscope98 authored Oct 18, 2024
1 parent ef65962 commit 5935b25
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 34 deletions.
68 changes: 34 additions & 34 deletions .github/workflows/llm_performance_tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -783,7 +783,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -814,7 +814,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -845,7 +845,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -882,7 +882,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -916,7 +916,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\32-32_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\32-32_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
REM if %ERRORLEVEL% neq 0 if %ERRORLEVEL% neq -1073740791 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -971,7 +971,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1002,7 +1002,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1033,7 +1033,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1070,7 +1070,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1104,7 +1104,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1158,7 +1158,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1189,7 +1189,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1220,7 +1220,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1257,7 +1257,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1291,7 +1291,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\2048-256_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\2048-256_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1345,7 +1345,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1379,7 +1379,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_436.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1415,7 +1415,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
if "${{ matrix.platform }}"=="perf-igpu" (
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
)
Expand Down Expand Up @@ -1462,7 +1462,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
if "${{ matrix.platform }}"=="perf-igpu" (
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
)
Expand Down Expand Up @@ -1506,7 +1506,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\3072-384_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\3072-384_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
if "${{ matrix.platform }}"=="perf-igpu" (
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
)
Expand Down Expand Up @@ -1570,7 +1570,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1601,7 +1601,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_438.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1637,7 +1637,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_443.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1671,7 +1671,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\4096-512_int4_fp16_445.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\4096-512_int4_fp16\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1727,7 +1727,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1760,7 +1760,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_436.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1793,7 +1793,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_438.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1827,7 +1827,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_443.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1863,7 +1863,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_loadlowbit_445.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16_loadlowbit\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1919,7 +1919,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test1
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1952,7 +1952,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_436.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test2
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -1985,7 +1985,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_438.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test3
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -2019,7 +2019,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_443.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test4
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down Expand Up @@ -2055,7 +2055,7 @@ jobs:
move ..\..\..\test\benchmark\igpu-perf\1024-128_445.yaml config.yaml
set PYTHONIOENCODING=utf-8
python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
if %ERRORLEVEL% neq 0 (exit /b 1)
REM if %ERRORLEVEL% neq 0 (exit /b 1)
python ..\..\..\test\benchmark\igpu-perf\check_csv_results.py --yaml-file config.yaml --suffix test5
if %ERRORLEVEL% neq 0 (exit /b 1)
Expand Down
2 changes: 2 additions & 0 deletions python/llm/test/benchmark/igpu-perf/check_csv_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ def main():
if test_case not in actual_test_cases and test_case not in exclude_test_cases:
print(test_case)
raise ValueError("The above tests failed. Please check the errors in the log.")
else:
print("Results integrity check completed")

if __name__ == "__main__":
sys.exit(main())

0 comments on commit 5935b25

Please sign in to comment.