From 5c408ab48b6bed5d2e53c3526139559ce80e5300 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Tue, 4 Jun 2024 13:37:58 +0800 Subject: [PATCH] temp test --- .github/workflows/llm_performance_tests.yml | 591 ++++++++++---------- 1 file changed, 297 insertions(+), 294 deletions(-) diff --git a/.github/workflows/llm_performance_tests.yml b/.github/workflows/llm_performance_tests.yml index 9cea1ca18057..eeb58c34740a 100644 --- a/.github/workflows/llm_performance_tests.yml +++ b/.github/workflows/llm_performance_tests.yml @@ -13,12 +13,12 @@ on: schedule: - cron: "30 16 * * *" # GMT time, 16:30 GMT == 00:30 China # please uncomment it for PR tests - # pull_request: - # branches: [main] - # paths: - # - ".github/workflows/llm_performance_tests.yml" - # - "python/llm/test/benchmark/**" - # - "python/llm/dev/benchmark/all-in-one/**" + pull_request: + branches: [main] + paths: + - ".github/workflows/llm_performance_tests.yml" + - "python/llm/test/benchmark/**" + - "python/llm/dev/benchmark/all-in-one/**" workflow_dispatch: workflow_call: @@ -345,6 +345,7 @@ jobs: echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV" - name: Install ipex-llm and other related packages (install from pypi) + if: false shell: cmd run: | call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y @@ -366,6 +367,7 @@ jobs: call conda deactivate - name: Create env for html generation + if: false shell: cmd run: | call conda create -n html-gen python=3.11 -y @@ -377,6 +379,7 @@ jobs: call conda deactivate - name: Set directory envs & and fix generated csv date name + if: false shell: bash run: | if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then @@ -397,357 +400,357 @@ jobs: run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming) ' python/llm/dev/benchmark/all-in-one/run.py - - name: Prepare igpu perf test (32-32) - shell: bash - run: | - # hide time info - # sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32.yaml - - - name: Test on igpu (32-32) - shell: cmd - run: | - call conda activate igpu-perf - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 - REM for llava - set TRANSFORMERS_OFFLINE=1 + # - name: Prepare igpu perf test (32-32) + # shell: bash + # run: | + # # hide time info + # # sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32.yaml - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\32-32.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # - name: Test on igpu (32-32) + # shell: cmd + # run: | + # call conda activate igpu-perf + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 + # REM for llava + # set TRANSFORMERS_OFFLINE=1 + + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\32-32.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Prepare igpu perf test for transformers 4.37 (32-32) - shell: bash - run: | - sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_437.yaml + # - name: Prepare igpu perf test for transformers 4.37 (32-32) + # shell: bash + # run: | + # sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_437.yaml - - name: Test on igpu for transformers 4.37 (32-32) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.37.0 + # - name: Test on igpu for transformers 4.37 (32-32) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.37.0 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\32-32_437.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\32-32_437.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Concat csv and generate html (32-32) - shell: cmd - run: | - call conda activate html-gen + # - name: Concat csv and generate html (32-32) + # shell: cmd + # run: | + # call conda activate html-gen - cd python\llm\dev\benchmark\all-in-one - python ..\..\..\test\benchmark\concat_csv.py - if %ERRORLEVEL% neq 0 (exit /b 1) - del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\32-32\ - cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\32-32\ - if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\32-32\*.html %CSV_SAVE_PATH% + # cd python\llm\dev\benchmark\all-in-one + # python ..\..\..\test\benchmark\concat_csv.py + # if %ERRORLEVEL% neq 0 (exit /b 1) + # del /q *test*.csv + # move *.csv %CSV_SAVE_PATH%\32-32\ + # cd ..\..\..\test\benchmark + # python csv_to_html.py -f %CSV_SAVE_PATH%\32-32\ + # if %ERRORLEVEL% neq 0 (exit /b 1) + # move %CSV_SAVE_PATH%\32-32\*.html %CSV_SAVE_PATH% - call conda deactivate + # call conda deactivate - # TODO: create a action function here for different input - # 1024-128 - - name: Prepare igpu perf test (1024-128) - shell: bash - run: | - sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml + # # TODO: create a action function here for different input + # # 1024-128 + # - name: Prepare igpu perf test (1024-128) + # shell: bash + # run: | + # sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml - - name: Test on igpu (1024-128) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.36.2 + # - name: Test on igpu (1024-128) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.36.2 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 - REM for llava - set TRANSFORMERS_OFFLINE=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 + # REM for llava + # set TRANSFORMERS_OFFLINE=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Prepare igpu perf test for transformers 4.37 (1024-128) - shell: bash - run: | - sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml + # - name: Prepare igpu perf test for transformers 4.37 (1024-128) + # shell: bash + # run: | + # sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml - - name: Test on igpu for transformers 4.37 (1024-128) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.37.0 + # - name: Test on igpu for transformers 4.37 (1024-128) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.37.0 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Concat csv and generate html (1024-128) - shell: cmd - run: | - call conda activate html-gen + # - name: Concat csv and generate html (1024-128) + # shell: cmd + # run: | + # call conda activate html-gen - cd python\llm\dev\benchmark\all-in-one - python ..\..\..\test\benchmark\concat_csv.py - if %ERRORLEVEL% neq 0 (exit /b 1) - del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\1024-128\ - cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\ - if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH% + # cd python\llm\dev\benchmark\all-in-one + # python ..\..\..\test\benchmark\concat_csv.py + # if %ERRORLEVEL% neq 0 (exit /b 1) + # del /q *test*.csv + # move *.csv %CSV_SAVE_PATH%\1024-128\ + # cd ..\..\..\test\benchmark + # python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\ + # if %ERRORLEVEL% neq 0 (exit /b 1) + # move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH% - call conda deactivate + # call conda deactivate - # 2048-256 - - name: Prepare igpu perf test (2048-256) - shell: bash - run: | - sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml + # # 2048-256 + # - name: Prepare igpu perf test (2048-256) + # shell: bash + # run: | + # sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml - - name: Test on igpu (2048-256) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.36.2 + # - name: Test on igpu (2048-256) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.36.2 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 - REM for llava - set TRANSFORMERS_OFFLINE=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 + # REM for llava + # set TRANSFORMERS_OFFLINE=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Prepare igpu perf test for transformers 4.37 (2048-256) - shell: bash - run: | - sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_437.yaml + # - name: Prepare igpu perf test for transformers 4.37 (2048-256) + # shell: bash + # run: | + # sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_437.yaml - - name: Test on igpu for transformers 4.37 (2048-256) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.37.0 + # - name: Test on igpu for transformers 4.37 (2048-256) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.37.0 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\2048-256_437.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\2048-256_437.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Concat csv and generate html (2048-256) - shell: cmd - run: | - call conda activate html-gen + # - name: Concat csv and generate html (2048-256) + # shell: cmd + # run: | + # call conda activate html-gen - cd python\llm\dev\benchmark\all-in-one - python ..\..\..\test\benchmark\concat_csv.py - if %ERRORLEVEL% neq 0 (exit /b 1) - del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\2048-256\ - cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\ - if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH% + # cd python\llm\dev\benchmark\all-in-one + # python ..\..\..\test\benchmark\concat_csv.py + # if %ERRORLEVEL% neq 0 (exit /b 1) + # del /q *test*.csv + # move *.csv %CSV_SAVE_PATH%\2048-256\ + # cd ..\..\..\test\benchmark + # python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\ + # if %ERRORLEVEL% neq 0 (exit /b 1) + # move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH% - call conda deactivate + # call conda deactivate - # load_low_bit 1024-128 - - name: Prepare igpu perf test (load_low_bit 1024-128) - shell: bash - run: | - sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml + # # load_low_bit 1024-128 + # - name: Prepare igpu perf test (load_low_bit 1024-128) + # shell: bash + # run: | + # sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml - - name: Test on igpu (load_low_bit 1024-128) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.36.2 + # - name: Test on igpu (load_low_bit 1024-128) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.36.2 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 - REM for llava - set TRANSFORMERS_OFFLINE=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 + # REM for llava + # set TRANSFORMERS_OFFLINE=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128) - shell: bash - run: | - sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml + # - name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128) + # shell: bash + # run: | + # sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml - - name: Test on igpu for transformers 4.37 (load_low_bit 1024-128) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.37.0 + # - name: Test on igpu for transformers 4.37 (load_low_bit 1024-128) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.37.0 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_437.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_437.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Concat csv and generate html (load_low_bit 1024-128) - shell: cmd - run: | - call conda activate html-gen + # - name: Concat csv and generate html (load_low_bit 1024-128) + # shell: cmd + # run: | + # call conda activate html-gen - cd python\llm\dev\benchmark\all-in-one - python ..\..\..\test\benchmark\concat_csv.py - if %ERRORLEVEL% neq 0 (exit /b 1) - del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\1024-128_loadlowbit\ - cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_loadlowbit\ - if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\1024-128_loadlowbit\*.html %CSV_SAVE_PATH% + # cd python\llm\dev\benchmark\all-in-one + # python ..\..\..\test\benchmark\concat_csv.py + # if %ERRORLEVEL% neq 0 (exit /b 1) + # del /q *test*.csv + # move *.csv %CSV_SAVE_PATH%\1024-128_loadlowbit\ + # cd ..\..\..\test\benchmark + # python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_loadlowbit\ + # if %ERRORLEVEL% neq 0 (exit /b 1) + # move %CSV_SAVE_PATH%\1024-128_loadlowbit\*.html %CSV_SAVE_PATH% - call conda deactivate + # call conda deactivate - - name: Prepare igpu perf test (int4+fp16 1024-128) - shell: bash - run: | - sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml + # - name: Prepare igpu perf test (int4+fp16 1024-128) + # shell: bash + # run: | + # sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml - - name: Test on igpu (int4+fp16 1024-128) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.36.2 + # - name: Test on igpu (int4+fp16 1024-128) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.36.2 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 - REM for llava - set TRANSFORMERS_OFFLINE=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 + # REM for llava + # set TRANSFORMERS_OFFLINE=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Prepare igpu perf test for transformers 4.37 (int4+fp16 1024-128) - shell: bash - run: | - sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py - sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml + # - name: Prepare igpu perf test for transformers 4.37 (int4+fp16 1024-128) + # shell: bash + # run: | + # sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + # sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml - - name: Test on igpu for transformers 4.37 (int4+fp16 1024-128) - shell: cmd - run: | - call conda activate igpu-perf - pip install transformers==4.37.0 + # - name: Test on igpu for transformers 4.37 (int4+fp16 1024-128) + # shell: cmd + # run: | + # call conda activate igpu-perf + # pip install transformers==4.37.0 - set SYCL_CACHE_PERSISTENT=1 - set BIGDL_LLM_XMX_DISABLED=1 + # set SYCL_CACHE_PERSISTENT=1 + # set BIGDL_LLM_XMX_DISABLED=1 - cd python\llm\dev\benchmark\all-in-one - move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml - set PYTHONIOENCODING=utf-8 - python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 - if %ERRORLEVEL% neq 0 (exit /b 1) + # cd python\llm\dev\benchmark\all-in-one + # move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml + # set PYTHONIOENCODING=utf-8 + # python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1 + # if %ERRORLEVEL% neq 0 (exit /b 1) - call conda deactivate + # call conda deactivate - - name: Concat csv and generate html (int4+fp16 1024-128) - shell: cmd - run: | - call conda activate html-gen + # - name: Concat csv and generate html (int4+fp16 1024-128) + # shell: cmd + # run: | + # call conda activate html-gen - cd python\llm\dev\benchmark\all-in-one - python ..\..\..\test\benchmark\concat_csv.py - if %ERRORLEVEL% neq 0 (exit /b 1) - del /q *test*.csv - move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\ - cd ..\..\..\test\benchmark - python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\ - if %ERRORLEVEL% neq 0 (exit /b 1) - move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH% + # cd python\llm\dev\benchmark\all-in-one + # python ..\..\..\test\benchmark\concat_csv.py + # if %ERRORLEVEL% neq 0 (exit /b 1) + # del /q *test*.csv + # move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\ + # cd ..\..\..\test\benchmark + # python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\ + # if %ERRORLEVEL% neq 0 (exit /b 1) + # move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH% - call conda deactivate + # call conda deactivate - # TODO: avoid duplicated code - - name: Upload results to ftp - if: ${{ always() }} - shell: cmd - run: | - cd %CSV_SAVE_PATH% - IF "${{ github.event_name }}"=="schedule" ( - for %%f in (*.html) do ( - curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH% - ) - ) - IF "${{ github.event_name }}"=="workflow_dispatch" ( - for %%f in (*.html) do ( - curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH% - ) - ) - # for test on machine when encountering error - # - name: Remove conda env + # # TODO: avoid duplicated code + # - name: Upload results to ftp # if: ${{ always() }} # shell: cmd # run: | - # call conda env remove -n igpu-perf -y \ No newline at end of file + # cd %CSV_SAVE_PATH% + # IF "${{ github.event_name }}"=="schedule" ( + # for %%f in (*.html) do ( + # curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH% + # ) + # ) + # IF "${{ github.event_name }}"=="workflow_dispatch" ( + # for %%f in (*.html) do ( + # curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH% + # ) + # ) + # # for test on machine when encountering error + # # - name: Remove conda env + # # if: ${{ always() }} + # # shell: cmd + # # run: | + # # call conda env remove -n igpu-perf -y \ No newline at end of file