From 6cee70a8dd06af9012bdd7a5ed675ca0f7a60e61 Mon Sep 17 00:00:00 2001 From: liu-shaojun Date: Fri, 8 Mar 2024 14:17:55 +0800 Subject: [PATCH] enable tests for ipex-llm --- .github/workflows/ipex-llm-nightly-test.yml | 118 +++ .github/workflows/ipex_llm_example_tests.yml | 80 ++ .../workflows/ipex_llm_performance_tests.yml | 791 ++++++++++++++++++ .github/workflows/ipex_llm_unit_tests.yml | 398 +++++++++ 4 files changed, 1387 insertions(+) create mode 100644 .github/workflows/ipex-llm-nightly-test.yml create mode 100644 .github/workflows/ipex_llm_example_tests.yml create mode 100644 .github/workflows/ipex_llm_performance_tests.yml create mode 100644 .github/workflows/ipex_llm_unit_tests.yml diff --git a/.github/workflows/ipex-llm-nightly-test.yml b/.github/workflows/ipex-llm-nightly-test.yml new file mode 100644 index 00000000000..68f96f9c263 --- /dev/null +++ b/.github/workflows/ipex-llm-nightly-test.yml @@ -0,0 +1,118 @@ +name: LLM Nightly Tests + +# Cancel previous runs in the PR when you push new commits +concurrency: + group: ${{ github.workflow }}-llm-nightly-test-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +# Controls when the action will run. +on: + schedule: + - cron: "00 13 * * *" # GMT time, 13:00 GMT == 21:00 China + pull_request: + branches: [main] + # paths: + # - ".github/workflows/llm-nightly-test.yml" + # - ".github/actions/llm/setup-llm-env/action.yml" + # - ".github/actions/llm/remove-llm-env/action.yml" + # - ".github/actions/llm/convert-test/action.yml" + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # llm-cpp-build: + # uses: ./.github/workflows/llm-binary-build.yml + llm-nightly-convert-test: + # needs: llm-cpp-build + strategy: + fail-fast: false + matrix: + include: + - os: windows + instruction: AVX-VNNI-UT + python-version: "3.9" + - os: ubuntu-20.04-lts + instruction: avx512 + python-version: "3.9" + runs-on: [self-hosted, llm, "${{matrix.instruction}}", "${{matrix.os}}"] + env: + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + steps: + - name: Set model directories + shell: bash + run: | + echo "ORIGIN_DIR=$(pwd)/../llm/origin-models" >> "$GITHUB_ENV" + echo "INT4_CKPT_DIR=$(pwd)/../llm/nightly-converted-models" >> "$GITHUB_ENV" + - name: Create model directories + shell: bash + run: | + if [ ! -d $ORIGIN_DIR ]; then + mkdir -p $ORIGIN_DIR + fi + if [ ! -d $INT4_CKPT_DIR ]; then + mkdir -p $INT4_CKPT_DIR + fi + - name: Set environment variables + shell: bash + run: | + echo "LLAMA_ORIGIN_PATH=${ORIGIN_DIR}/llama-7b-hf" >> "$GITHUB_ENV" + echo "GPTNEOX_ORIGIN_PATH=${ORIGIN_DIR}/gptneox-7b-redpajama-bf16" >> "$GITHUB_ENV" + echo "BLOOM_ORIGIN_PATH=${ORIGIN_DIR}/bloomz-7b1" >> "$GITHUB_ENV" + echo "STARCODER_ORIGIN_PATH=${ORIGIN_DIR}/gpt_bigcode-santacoder" >> "$GITHUB_ENV" + + echo "LLAMA_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_llama_q4_0.bin" >> "$GITHUB_ENV" + echo "GPTNEOX_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_gptneox_q4_0.bin" >> "$GITHUB_ENV" + echo "BLOOM_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_bloom_q4_0.bin" >> "$GITHUB_ENV" + echo "STARCODER_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_starcoder_q4_0.bin" >> "$GITHUB_ENV" + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools==58.0.4 + python -m pip install --upgrade wheel + + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Install BigDL-LLM + # uses: ./.github/actions/llm/setup-llm-env + + - name: Install IPEX-LLM from Pypi + shell: bash + run: | + pip install --pre --upgrade ipex-llm[all] + + - name: Download original models & convert + uses: ./.github/actions/llm/convert-test + + - name: Upload ckpt to ftp + shell: bash + if: runner.os == 'Linux' && github.event_name == 'schedule' + run: | + curl -T $LLAMA_INT4_CKPT_PATH ${LLM_FTP_URL}/llm/ggml-actions/nightly/bigdl_llm_llama_7b_q4_0.bin + curl -T $GPTNEOX_INT4_CKPT_PATH ${LLM_FTP_URL}/llm/ggml-actions/nightly/bigdl_llm_redpajama_7b_q4_0.bin + curl -T $BLOOM_INT4_CKPT_PATH ${LLM_FTP_URL}/llm/ggml-actions/nightly/bigdl_llm_bloom_7b_q4_0.bin + curl -T $STARCODER_INT4_CKPT_PATH ${LLM_FTP_URL}/llm/ggml-actions/nightly/bigdl_llm_santacoder_1b_q4_0.bin + - name: Delete ckpt + shell: bash + run: | + rm -rf $LLAMA_INT4_CKPT_PATH + rm -rf $GPTNEOX_INT4_CKPT_PATH + rm -rf $BLOOM_INT4_CKPT_PATH + rm -rf $STARCODER_INT4_CKPT_PATH + + llm-unit-tests: + # needs: llm-cpp-build + uses: ./.github/workflows/ipex_llm_unit_tests.yml + llm-example-test: + # needs: llm-cpp-build + uses: ./.github/workflows/ipex_llm_example_tests.yml diff --git a/.github/workflows/ipex_llm_example_tests.yml b/.github/workflows/ipex_llm_example_tests.yml new file mode 100644 index 00000000000..386a6255812 --- /dev/null +++ b/.github/workflows/ipex_llm_example_tests.yml @@ -0,0 +1,80 @@ +name: LLM Example Test + +# Cancel previous runs in the PR when you push new commits +concurrency: + group: ${{ github.workflow }}-llm-example-tests-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +# Controls when the action will run. +on: + # schedule: + # - cron: '00 13 * * *' # GMT time, 13:00 GMT == 21:00 China + pull_request: + branches: [ main ] + # paths: + # - '.github/workflows/llm_example_tests.yml' + # - '.github/workflows/llm-binary-build.yml' + # - '.github/actions/llm/example-test/action.yml' + # - '.github/actions/llm/setup-llm-env/action.yml' + # - '.github/actions/llm/remove-llm-env/action.yml' + # - '.github/actions/llm/download-llm-binary/action.yml' + # - 'python/llm/dev/test/run-example-tests.sh' + # - 'python/llm/example/**' + workflow_dispatch: + workflow_call: + +env: + INT4_CKPT_DIR: ./llm/ggml-actions/stable + LLM_DIR: ./llm + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # llm-cpp-build: + # uses: ./.github/workflows/llm-binary-build.yml + llm-example-test: + # needs: llm-cpp-build + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + instruction: ["AVX512"] + runs-on: [ self-hosted, llm,"${{matrix.instruction}}", ubuntu-20.04-lts ] + env: + THREAD_NUM: 24 + steps: + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools==58.0.4 + python -m pip install --upgrade wheel + + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Run LLM install (all) test + # uses: ./.github/actions/llm/setup-llm-env + # env: + # ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + + - name: Install IPEX-LLM from Pypi + shell: bash + run: | + pip install --pre --upgrade ipex-llm[all] + + - name: Run LLM example test + uses: ./.github/actions/llm/example-test + env: + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + + # - name: Clean up test environment + # uses: ./.github/actions/llm/remove-llm-env + # env: + # ANALYTICS_ZOO_ROOT: ${{ github.workspace }} diff --git a/.github/workflows/ipex_llm_performance_tests.yml b/.github/workflows/ipex_llm_performance_tests.yml new file mode 100644 index 00000000000..68b9cbbd2cc --- /dev/null +++ b/.github/workflows/ipex_llm_performance_tests.yml @@ -0,0 +1,791 @@ +name: LLM Performance Test + +# Cancel previous runs in the PR when you push new commits +concurrency: + group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +# Controls when the action will run. +on: + schedule: + - cron: "30 16 * * *" # GMT time, 16:30 GMT == 00:30 China + # please uncomment it for PR tests + pull_request: + branches: [main] + # paths: + # - ".github/workflows/llm_performance_tests.yml" + # - "python/llm/test/benchmark/**" + # - "python/llm/dev/benchmark/all-in-one/**" + workflow_dispatch: + workflow_call: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # llm-cpp-build: # please uncomment it for PR tests + # uses: ./.github/workflows/llm-binary-build.yml + + llm-performance-test-on-arc: + # if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-arc' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests + # needs: llm-cpp-build # please uncomment it for PR tests + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + runs-on: [self-hosted, llm, perf] + env: + OMP_NUM_THREADS: 16 + THREAD_NUM: 16 + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + CSV_SAVE_PATH: ${{ github.event.schedule && '/mnt/disk1/nightly_perf_gpu/' || '/mnt/disk1/pr_perf_gpu/' }} + + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + shell: bash + # pip install transformers_stream_generator for model internlm-chat-7b-8k + # pip install tiktoken for model Qwen-7B-Chat-10-12 + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade wheel + python -m pip install --upgrade omegaconf + python -m pip install --upgrade pandas + python -m pip install --upgrade einops + python -m pip install --upgrade transformers_stream_generator + python -m pip install --upgrade tiktoken + + # please uncomment it and comment the "Install BigDL-LLM from Pypi" part for PR tests + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Run LLM install (all) test + # uses: ./.github/actions/llm/setup-llm-env + # with: + # extra-dependency: "xpu_2.1" + + - name: Install BigDL-LLM from Pypi + shell: bash + run: | + pip install --pre --upgrade ipex-llm[xpu] -f https://developer.intel.com/ipex-whl-stable-xpu + # test_version_date=`date -d 'yesterday' '+%Y%m%d'` + # if ! pip show bigdl-llm | grep $test_version_date; then + # echo "Did not install bigdl-llm with excepted version $test_version_date" + # exit 1 + # fi + + - name: Test installed xpu version + shell: bash + run: | + source /opt/intel/oneapi/setvars.sh + bash python/llm/test/run-llm-install-tests.sh + + - name: Test on xpu(transformers==4.31.0) + shell: bash + run: | + date_for_test_version=$(date -d yesterday +%Y-%m-%d) + sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py + + source /opt/intel/oneapi/setvars.sh + export USE_XETLA=OFF + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + # hide time info + sed -i 's/str(end - st)/"xxxxxx"/g' run.py + # change csv name + sed -i 's/{today}/{today}_test1/g' run.py + python run.py + + - name: Test on xpu(transformers==4.34.0) + shell: bash + run: | + source /opt/intel/oneapi/setvars.sh + export USE_XETLA=OFF + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + # upgrade transformers for model Mistral-7B-v0.1 + python -m pip install transformers==4.34.0 + cp python/llm/test/benchmark/arc-perf-transformers-434.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + # change csv name + sed -i 's/test1/test2/g' run.py + python run.py + + - name: Test on xpu(transformers==4.37.0) + shell: bash + run: | + source /opt/intel/oneapi/setvars.sh + export USE_XETLA=OFF + export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + # upgrade transformers for model Qwen/Qwen1.5-7B-Chat + python -m pip install transformers==4.37.0 + cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + # change csv name + sed -i 's/test2/test3/g' run.py + python run.py + + - name: Concat csv and generate html + shell: bash + run: | + cd python/llm/dev/benchmark/all-in-one + python ../../../test/benchmark/concat_csv.py + for file in *.csv; do + if [[ $file != *test* ]]; then + cp "$file" $CSV_SAVE_PATH + fi + done + python -m pip install pandas==1.5.3 + cd ../../../test/benchmark + python csv_to_html.py -f $CSV_SAVE_PATH + + - name: Check and upload results to ftp + shell: bash + run: | + cd python/llm/dev/benchmark/all-in-one + python ../../../test/benchmark/check_results.py -c test1 -y ../../../test/benchmark/arc-perf-test.yaml + python ../../../test/benchmark/check_results.py -c test2 -y ../../../test/benchmark/arc-perf-transformers-434.yaml + find . -name "*test*.csv" -delete + if [ ${{ github.event.schedule}} ]; then + curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/ + fi + + llm-performance-test-on-spr: + # if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-spr' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests + # needs: llm-cpp-build # please uncomment it for PR tests + strategy: + fail-fast: false + matrix: + python-version: ["3.9"] + runs-on: [self-hosted, llm, spr01-perf] + env: + OMP_NUM_THREADS: 16 + THREAD_NUM: 16 + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade wheel + python -m pip install --upgrade omegaconf + python -m pip install --upgrade pandas + python -m pip install --upgrade einops + python -m pip install --upgrade tiktoken + python -m pip install --upgrade transformers_stream_generator + + # please uncomment it and comment the "Install BigDL-LLM from Pypi" part for PR tests + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Run LLM install (all) test + # uses: ./.github/actions/llm/setup-llm-env + + - name: Install BigDL-LLM from Pypi + shell: bash + run: | + pip install --pre --upgrade ipex-llm[all] -f https://developer.intel.com/ipex-whl-stable-xpu + # test_version_date=`date -d 'yesterday' '+%Y%m%d'` + # if ! pip show bigdl-llm | grep $test_version_date; then + # echo "Did not install bigdl-llm with excepted version $test_version_date" + # exit 1 + # fi + + - name: Test on cpu + shell: bash + run: | + date_for_test_version=$(date -d yesterday +%Y-%m-%d) + sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py + + mv python/llm/test/benchmark/cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + export http_proxy=${HTTP_PROXY} + export https_proxy=${HTTPS_PROXY} + source bigdl-llm-init -t + export OMP_NUM_THREADS=48 + # hide time info + sed -i 's/str(end - st)/"xxxxxx"/g' run.py + python run.py + cp ./*.csv /models/nightly_perf_cpu + cd ../../../test/benchmark + python -m pip install pandas==1.5.3 + python csv_to_html.py -f /models/nightly_perf_cpu + cd /models/nightly_perf_cpu + for f in *.html; do + curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/ + done + + llm-performance-test-on-core: + # if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-core' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests + # needs: llm-cpp-build # please uncomment it for PR tests + strategy: + fail-fast: false + matrix: + include: + - os: windows + platform: dp + python-version: "3.9" + # - os: windows + # platform: lp + # python-version: "3.9" + runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"] + env: + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + CSV_SAVE_PATH: ${{ github.event.schedule && 'D:/action-runners/nightly_perf_core_' || 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/ + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade wheel + python -m pip install --upgrade omegaconf pandas + python -m pip install --upgrade tiktoken einops transformers_stream_generator + + # please uncomment it and comment the "Install BigDL-LLM from Pypi" part for PR tests + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Run LLM install (all) test + # uses: ./.github/actions/llm/setup-llm-env + + - name: Install BigDL-LLM from Pypi + shell: bash + run: | + pip install --pre --upgrade ipex-llm[all] -f https://developer.intel.com/ipex-whl-stable-xpu + # test_version_date=`date -d 'yesterday' '+%Y%m%d'` + # if ! pip show bigdl-llm | grep $test_version_date; then + # echo "Did not install bigdl-llm with excepted version $test_version_date" + # exit 1 + # fi + + - name: Test on core ${{ matrix.platform }} + shell: bash + run: | + date_for_test_version=$(date -d yesterday +%Y-%m-%d) + sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py + + mv python/llm/test/benchmark/core-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml + cd python/llm/dev/benchmark/all-in-one + export http_proxy=${HTTP_PROXY} + export https_proxy=${HTTPS_PROXY} + # hide time info + sed -i 's/str(end - st)/"xxxxxx"/g' run.py + python run.py + cp ./*.csv $CSV_SAVE_PATH + cd ../../../test/benchmark + python -m pip install pandas==1.5.3 + python csv_to_html.py -f $CSV_SAVE_PATH + cd ../../dev/benchmark/all-in-one/ + if [ ${{ github.event.schedule}} ]; then + curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/ + fi + + llm-performance-test-on-igpu: + # if: ${{ github.event.schedule || github.event.inputs.artifact == 'llm-performance-test-on-igpu' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests + # needs: llm-cpp-build # please uncomment it for PR tests + strategy: + fail-fast: false + matrix: + include: + - os: windows + python-version: "3.9" + runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-igpu] + env: + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 + + # TODO: Put the bigdl-llm related install process for win gpu into a action function + + # Please uncomment it and commment the install from pypi for PR tests + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Prepare for install bigdl-llm from source + # shell: bash + # run: | + # sed -i 's/"bigdl-core-xe-21==" + VERSION + "/"bigdl-core-xe-21/g' python/llm/setup.py + # sed -i 's/"bigdl-core-xe-21==" + VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py + + # - name: Install bigdl-llm and other related packages (install from source) + # shell: cmd + # run: | + # call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y + # call conda activate igpu-perf + + # pip install --upgrade pip + # pip install --upgrade wheel + # pip install --upgrade omegaconf pandas + # pip install --upgrade tiktoken einops transformers_stream_generator + + # cd python\llm + # python setup.py clean --all bdist_wheel --win + # if not exist dist\bigdl_llm*.whl (exit /b 1) + # for %%i in (dist\bigdl_llm*.whl) do set whl_name=%%i + + # pip install --pre --upgrade %whl_name%[xpu] -f https://developer.intel.com/ipex-whl-stable-xpu + # if %ERRORLEVEL% neq 0 (exit /b 1) + # pip list + + # call conda deactivate + + - name: Determine desired bigdl-llm version + shell: bash + run: | + test_version_date=`date -d 'yesterday' '+%Y%m%d'` + echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV" + + - name: Install bigdl-llm and other related packages (install from pypi) + shell: cmd + run: | + call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y + call conda activate igpu-perf + + pip install --upgrade pip + pip install --upgrade wheel + pip install --upgrade omegaconf pandas + pip install --upgrade tiktoken einops transformers_stream_generator + + pip install --pre --upgrade ipex-llm[xpu] -f https://developer.intel.com/ipex-whl-stable-xpu + # pip show bigdl-llm | findstr %TEST_VERSION_DATE% + # if %ERRORLEVEL% neq 0 ( + # echo "Did not install bigdl-llm with excepted version %TEST_VERSION_DATE%" + # exit /b 1 + # ) + pip list + + call conda deactivate + + - name: Create env for html generation + shell: cmd + run: | + call conda create -n html-gen python=3.9 -y + call conda activate html-gen + + pip install pandas==1.5.3 + pip install Jinja2 + + call conda deactivate + + - name: Set directory envs & and fix generated csv date name + shell: bash + run: | + if [ ${{ github.event_name }} == 'schedule' ]; then + echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV" + else + echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV" + fi + date_for_test_version=$(date -d yesterday +%Y-%m-%d) + echo "LOG_FILE=${date_for_test_version}_output.txt" >> "$GITHUB_ENV" + + sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py + + - name: Prepare igpu perf test (32-32) + shell: bash + run: | + # hide time info + # sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32.yaml + + - name: Test on igpu (32-32) + shell: cmd + run: | + call conda activate igpu-perf + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + REM for llava + set TRANSFORMERS_OFFLINE=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\32-32.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Mistral (32-32) + shell: bash + run: | + sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_434.yaml + + - name: Test on igpu for Mistral (32-32) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.34.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\32-32_434.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Qwen1.5 (32-32) + shell: bash + run: | + sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_437.yaml + + - name: Test on igpu for Qwen1.5 (32-32) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.37.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\32-32_437.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Concat csv and generate html (32-32) + shell: cmd + run: | + call conda activate html-gen + + cd python\llm\dev\benchmark\all-in-one + python ..\..\..\test\benchmark\concat_csv.py + if %ERRORLEVEL% neq 0 (exit /b 1) + del /q *test*.csv + move *.csv %CSV_SAVE_PATH%\32-32\ + cd ..\..\..\test\benchmark + python csv_to_html.py -f %CSV_SAVE_PATH%\32-32\ + if %ERRORLEVEL% neq 0 (exit /b 1) + move %CSV_SAVE_PATH%\32-32\*.html %CSV_SAVE_PATH% + + call conda deactivate + + # TODO: create a action function here for different input + # 1024-128 + - name: Prepare igpu perf test (1024-128) + shell: bash + run: | + sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml + + - name: Test on igpu (1024-128) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.31.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + REM for llava + set TRANSFORMERS_OFFLINE=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Mistral (1024-128) + shell: bash + run: | + sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_434.yaml + + - name: Test on igpu for Mistral (1024-128) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.34.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_434.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Qwen 1.5 (1024-128) + shell: bash + run: | + sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml + + - name: Test on igpu for Qwen 1.5 (1024-128) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.37.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Concat csv and generate html (1024-128) + shell: cmd + run: | + call conda activate html-gen + + cd python\llm\dev\benchmark\all-in-one + python ..\..\..\test\benchmark\concat_csv.py + if %ERRORLEVEL% neq 0 (exit /b 1) + del /q *test*.csv + move *.csv %CSV_SAVE_PATH%\1024-128\ + cd ..\..\..\test\benchmark + python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\ + if %ERRORLEVEL% neq 0 (exit /b 1) + move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH% + + call conda deactivate + + # 2048-256 + - name: Prepare igpu perf test (2048-256) + shell: bash + run: | + sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml + + - name: Test on igpu (2048-256) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.31.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + REM for llava + set TRANSFORMERS_OFFLINE=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Mistral (2048-256) + shell: bash + run: | + sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_434.yaml + + - name: Test on igpu for Mistral (2048-256) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.34.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\2048-256_434.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Qwen 1.5 (2048-256) + shell: bash + run: | + sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_437.yaml + + - name: Test on igpu for Qwen 1.5 (2048-256) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.37.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\2048-256_437.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Concat csv and generate html (2048-256) + shell: cmd + run: | + call conda activate html-gen + + cd python\llm\dev\benchmark\all-in-one + python ..\..\..\test\benchmark\concat_csv.py + if %ERRORLEVEL% neq 0 (exit /b 1) + del /q *test*.csv + move *.csv %CSV_SAVE_PATH%\2048-256\ + cd ..\..\..\test\benchmark + python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\ + if %ERRORLEVEL% neq 0 (exit /b 1) + move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH% + + call conda deactivate + + # load_low_bit 1024-128 + - name: Prepare igpu perf test (load_low_bit 1024-128) + shell: bash + run: | + sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i 's/{today}_test3/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml + + - name: Test on igpu (load_low_bit 1024-128) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.31.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + REM for llava + set TRANSFORMERS_OFFLINE=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Mistral (load_low_bit 1024-128) + shell: bash + run: | + sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_434.yaml + + - name: Test on igpu for Mistral (load_low_bit 1024-128) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.34.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_434.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Prepare igpu perf test for Qwen 1.5 (load_low_bit 1024-128) + shell: bash + run: | + sed -i 's/{today}_test2/{today}_test3/g' python/llm/dev/benchmark/all-in-one/run.py + sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml + + - name: Test on igpu for Qwen 1.5 (load_low_bit 1024-128) + shell: cmd + run: | + call conda activate igpu-perf + pip install transformers==4.37.0 + + call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" + set SYCL_CACHE_PERSISTENT=1 + set BIGDL_LLM_XMX_DISABLED=1 + + cd python\llm\dev\benchmark\all-in-one + move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_437.yaml config.yaml + set PYTHONIOENCODING=utf-8 + python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1 + if %ERRORLEVEL% neq 0 (exit /b 1) + + call conda deactivate + + - name: Concat csv and generate html (load_low_bit 1024-128) + shell: cmd + run: | + call conda activate html-gen + + cd python\llm\dev\benchmark\all-in-one + python ..\..\..\test\benchmark\concat_csv.py + if %ERRORLEVEL% neq 0 (exit /b 1) + del /q *test*.csv + move *.csv %CSV_SAVE_PATH%\1024-128_loadlowbit\ + cd ..\..\..\test\benchmark + python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_loadlowbit\ + if %ERRORLEVEL% neq 0 (exit /b 1) + move %CSV_SAVE_PATH%\1024-128_loadlowbit\*.html %CSV_SAVE_PATH% + + call conda deactivate + + - name: Upload results to ftp + if: ${{ always() }} + shell: cmd + run: | + cd %CSV_SAVE_PATH% + IF "${{ github.event_name }}"=="schedule" ( + for %%f in (*.html) do ( + curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH% + ) + ) + + # for test on machine when encountering error + # - name: Remove conda env + # if: ${{ always() }} + # shell: cmd + # run: | + # call conda env remove -n igpu-perf -y diff --git a/.github/workflows/ipex_llm_unit_tests.yml b/.github/workflows/ipex_llm_unit_tests.yml new file mode 100644 index 00000000000..3c3b1c233ce --- /dev/null +++ b/.github/workflows/ipex_llm_unit_tests.yml @@ -0,0 +1,398 @@ +name: LLM Unit Tests + +# Cancel previous runs in the PR when you push new commits +concurrency: + group: ${{ github.workflow }}-llm-unittest-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +# Controls when the action will run. +on: + # Triggers the workflow on push or pull request events but only for the main branch + push: + branches: [main] + paths: + - "python/llm/**" + - ".github/workflows/llm_unit_tests.yml" + - ".github/workflows/llm-binary-build.yml" + - ".github/actions/llm/setup-llm-env/action.yml" + - ".github/actions/llm/remove-llm-env/action.yml" + - ".github/actions/llm/cli-test-linux/action.yml" + - ".github/actions/llm/cli-test-windows/action.yml" + - ".github/actions/llm/download-llm-binary/action.yml" + pull_request: + branches: [main] + # paths: + # - "python/llm/**" + # - ".github/workflows/llm_unit_tests.yml" + # - ".github/workflows/llm-binary-build.yml" + # - ".github/actions/llm/setup-llm-env/action.yml" + # - ".github/actions/llm/remove-llm-env/action.yml" + # - ".github/actions/llm/cli-test-linux/action.yml" + # - ".github/actions/llm/cli-test-windows/action.yml" + # - ".github/actions/llm/download-llm-binary/action.yml" + workflow_dispatch: + workflow_call: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # llm-cpp-build: + # uses: ./.github/workflows/llm-binary-build.yml + setup-python-version: + runs-on: ubuntu-latest + outputs: + python-version: ${{ steps.setup-python-version.outputs.python-version }} + steps: + - name: setup-python-version + id: setup-python-version + run: | + if [ ${{ github.event_name }} == 'schedule' ]; then + python_version='["3.9", "3.10", "3.11"]' + else + python_version='["3.9"]' + fi + list=$(echo ${python_version} | jq -c) + echo "python-version=${list}" >> "$GITHUB_OUTPUT" + llm-unit-test: + needs: [setup-python-version] + strategy: + fail-fast: false + matrix: + os: [windows, ubuntu-20.04-lts] + python-version: ${{ fromJson(needs.setup-python-version.outputs.python-version) }} + include: + - os: windows + instruction: AVX-VNNI-UT + - os: ubuntu-20.04-lts + instruction: avx512 + runs-on: [self-hosted, llm, "${{matrix.instruction}}", "${{matrix.os}}"] + env: + THREAD_NUM: 24 + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + steps: + - name: Set model directories + shell: bash + run: | + echo "DATASET_DIR=${{ github.workspace }}/../llm/datasets" >> "$GITHUB_ENV" + echo "ORIGIN_DIR=${{ github.workspace }}/../llm/origin-models" >> "$GITHUB_ENV" + echo "INT4_CKPT_DIR=${{ github.workspace }}/../llm/converted-models" >> "$GITHUB_ENV" + - name: Create model directories + shell: bash + run: | + if [ ! -d $DATASET_DIR ]; then + mkdir -p $DATASET_DIR + fi + if [ ! -d $ORIGIN_DIR ]; then + mkdir -p $ORIGIN_DIR + fi + if [ ! -d $INT4_CKPT_DIR ]; then + mkdir -p $INT4_CKPT_DIR + fi + - name: Set environment variables + shell: bash + run: | + echo "SPEECH_DATASET_PATH=${DATASET_DIR}/librispeech_asr_dummy" >> "$GITHUB_ENV" + echo "COMMON_VOICE_PATH=${DATASET_DIR}/common_voice" >> "$GITHUB_ENV" + + echo "LLAMA_ORIGIN_PATH=${ORIGIN_DIR}/llama-7b-hf" >> "$GITHUB_ENV" + echo "BLOOM_ORIGIN_PATH=${ORIGIN_DIR}/bloom-7b1" >> "$GITHUB_ENV" + echo "ORIGINAL_CHATGLM2_6B_PATH=${ORIGIN_DIR}/chatglm2-6b" >> "$GITHUB_ENV" + echo "ORIGINAL_REPLIT_CODE_PATH=${ORIGIN_DIR}/replit-code-v1-3b" >> "$GITHUB_ENV" + echo "ORIGINAL_WHISPER_TINY_PATH=${ORIGIN_DIR}/whisper-tiny" >> "$GITHUB_ENV" + echo "MISTRAL_ORIGIN_PATH=${ORIGIN_DIR}/Mistral-7B-v0.1" >> "$GITHUB_ENV" + echo "LLAMA2_7B_ORIGIN_PATH=${ORIGIN_DIR}/Llama-2-7b-chat-hf" >> "$GITHUB_ENV" + + echo "LLAMA_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_llama_7b_q4_0.bin" >> "$GITHUB_ENV" + echo "GPTNEOX_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_redpajama_7b_q4_0.bin" >> "$GITHUB_ENV" + echo "BLOOM_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_bloom_7b_q4_0.bin" >> "$GITHUB_ENV" + echo "STARCODER_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_santacoder_1b_q4_0.bin" >> "$GITHUB_ENV" + echo "CHATGLM_INT4_CKPT_PATH=${INT4_CKPT_DIR}/chatglm2-6b-q4_0.bin" >> "$GITHUB_ENV" + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools==58.0.4 + python -m pip install --upgrade wheel + + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Run LLM install (all) test + # uses: ./.github/actions/llm/setup-llm-env + + - name: Install IPEX-LLM from Pypi + shell: bash + run: | + pip install --pre --upgrade ipex-llm[all] + + - name: Download ckpt & original models + shell: bash + run: | + if [ ! -e $LLAMA_INT4_CKPT_PATH ]; then + echo "Directory $LLAMA_INT4_CKPT_PATH not found. Downloading from FTP server..." + echo "wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_llama_7b_q4_0.bin -P $INT4_CKPT_DIR" + wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_llama_7b_q4_0.bin -P $INT4_CKPT_DIR + fi + if [ ! -e $GPTNEOX_INT4_CKPT_PATH ]; then + echo "Directory $GPTNEOX_INT4_CKPT_PATH not found. Downloading from FTP server..." + wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_redpajama_7b_q4_0.bin -P $INT4_CKPT_DIR + fi + if [ ! -e $BLOOM_INT4_CKPT_PATH ]; then + echo "Directory $BLOOM_INT4_CKPT_PATH not found. Downloading from FTP server..." + wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_bloom_7b_q4_0.bin -P $INT4_CKPT_DIR + fi + if [ ! -e $STARCODER_INT4_CKPT_PATH ]; then + echo "Directory $STARCODER_INT4_CKPT_PATH not found. Downloading from FTP server..." + wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_santacoder_1b_q4_0.bin -P $INT4_CKPT_DIR + fi + # if [ ! -e $CHATGLM_INT4_CKPT_PATH ]; then + # echo "Directory $CHATGLM_INT4_CKPT_PATH not found. Downloading from FTP server..." + # wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/chatglm2-6b-q4_0.bin -P $INT4_CKPT_DIR + # fi + if [ ! -d $ORIGINAL_CHATGLM2_6B_PATH ]; then + echo "Directory $ORIGINAL_CHATGLM2_6B_PATH not found. Downloading from FTP server..." + echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/chatglm2-6b -P $ORIGIN_DIR" + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/chatglm2-6b -P $ORIGIN_DIR + fi + if [ ! -d $ORIGINAL_REPLIT_CODE_PATH ]; then + echo "Directory $ORIGINAL_REPLIT_CODE_PATH not found. Downloading from FTP server..." + echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/replit-code-v1-3b -P $ORIGIN_DIR" + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/replit-code-v1-3b -P $ORIGIN_DIR + fi + if [ ! -d $ORIGINAL_WHISPER_TINY_PATH ]; then + echo "Directory $ORIGINAL_WHISPER_TINY_PATH not found. Downloading from FTP server..." + echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/whisper-tiny -P $ORIGIN_DIR" + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/whisper-tiny -P $ORIGIN_DIR + fi + if [ ! -d $MISTRAL_ORIGIN_PATH ]; then + echo "Directory $MISTRAL_ORIGIN_PATH not found. Downloading from FTP server..." + echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Mistral-7B-v0.1 -P $ORIGIN_DIR" + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Mistral-7B-v0.1 -P $ORIGIN_DIR + fi + if [ ! -d $LLAMA_ORIGIN_PATH ]; then + echo "Directory $LLAMA_ORIGIN_PATH not found. Downloading from FTP server..." + echo "wget --no-verbose $LLM_FTP_URL/llm/llama-7b-hf -P $ORIGIN_DIR" + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/llama-7b-hf -P $ORIGIN_DIR + fi + if [ ! -d $BLOOM_ORIGIN_PATH ]; then + echo "Directory $BLOOM_ORIGIN_PATH not found. Downloading from FTP server..." + echo "wget --no-verbose $LLM_FTP_URL/llm/bloom-7b1 -P $ORIGIN_DIR" + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/bloom-7b1 -P $ORIGIN_DIR + fi + if [ ! -d $SPEECH_DATASET_PATH ]; then + echo "Directory $SPEECH_DATASET_PATH not found. Downloading from FTP server..." + echo "wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/librispeech_asr_dummy -P $DATASET_DIR" + wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/librispeech_asr_dummy -P $DATASET_DIR + fi + if [ ! -d $COMMON_VOICE_PATH ]; then + echo "Directory $COMMON_VOICE_PATH not found. Downloading from FTP server..." + echo "wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/common_voice -P $DATASET_DIR" + wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/common_voice -P $DATASET_DIR + fi + if [ ! -d $LLAMA2_7B_ORIGIN_PATH ]; then + echo "Directory $LLAMA2_7B_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Llama-2-7b-chat-hf -P $ORIGIN_DIR + fi + + - name: Run LLM cli test (Linux) + if: runner.os == 'Linux' + uses: ./.github/actions/llm/cli-test-linux + - name: Run LLM cli test (Windows) + if: runner.os == 'Windows' + uses: ./.github/actions/llm/cli-test-windows + - name: Run LLM inference test + shell: bash + run: | + python -m pip install einops datasets librosa openai-whisper + bash python/llm/test/run-llm-inference-tests.sh + - name: Run LLM langchain test + shell: bash + run: | + pip install -U langchain==0.0.184 + pip install -U chromadb==0.3.25 + pip install -U pandas==2.0.3 + bash python/llm/test/run-llm-langchain-tests.sh + - name: Run LLM llamaindex test + shell: bash + run: | + pip install llama-index-readers-file llama-index-vector-stores-postgres llama-index-embeddings-huggingface + pip install transformers==4.31.0 + bash python/llm/test/run-llm-llamaindex-tests.sh + llm-unit-test-on-arc: + needs: [setup-python-version] + strategy: + fail-fast: false + matrix: + pytorch-version: ['2.1', '2.0'] + python-version: ${{ fromJson(needs.setup-python-version.outputs.python-version) }} + runs-on: [self-hosted, llm, arc-ut] + env: + # OMP_NUM_THREADS: 16 + # THREAD_NUM: 16 + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + steps: + - name: Set environment variables + shell: bash + run: | + echo "DATASET_DIR=${ORIGIN_DIR}/../datasets" >> "$GITHUB_ENV" + echo "ABIRATE_ENGLISH_QUOTES_PATH=${ORIGIN_DIR}/../datasets/abirate_english_quotes" >> "$GITHUB_ENV" + echo "SPEECH_DATASET_PATH=${ORIGIN_DIR}/../datasets/librispeech_asr_dummy" >> "$GITHUB_ENV" + + echo "LLAMA2_7B_ORIGIN_PATH=${ORIGIN_DIR}/Llama-2-7b-chat-hf" >> "$GITHUB_ENV" + echo "CHATGLM2_6B_ORIGIN_PATH=${ORIGIN_DIR}/chatglm2-6b" >> "$GITHUB_ENV" + echo "FALCON_7B_ORIGIN_PATH=${ORIGIN_DIR}/falcon-7b-instruct-with-patch" >> "$GITHUB_ENV" + echo "MPT_7B_ORIGIN_PATH=${ORIGIN_DIR}/mpt-7b-chat" >> "$GITHUB_ENV" + echo "WHISPER_TINY_ORIGIN_PATH=${ORIGIN_DIR}/whisper-tiny" >> "$GITHUB_ENV" + + echo "MISTRAL_7B_INSTRUCT_V0_1_ORIGIN_PATH=${ORIGIN_DIR}/Mistral-7B-Instruct-v0.1" >> "$GITHUB_ENV" + echo "BAICHUAN2_7B_ORIGIN_PATH=${ORIGIN_DIR}/Baichuan2-7B-Chat" >> "$GITHUB_ENV" + echo "QWEN_7B_ORIGIN_PATH=${ORIGIN_DIR}/Qwen-7B-Chat" >> "$GITHUB_ENV" + - name: Checkout repo + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools + python -m pip install --upgrade wheel + + # - name: Download llm binary + # uses: ./.github/actions/llm/download-llm-binary + + # - name: Install BigDL-LLM for xpu + # uses: ./.github/actions/llm/setup-llm-env + # with: + # extra-dependency: "xpu_${{ matrix.pytorch-version }}" + + - name: Install IPEX-LLM from Pypi + shell: bash + run: | + export xpu_version=xpu_${{ matrix.pytorch-version }} + pip install --pre --upgrade ipex-llm[$xpu_version] -f https://developer.intel.com/ipex-whl-stable-xpu + + - name: Test installed xpu version + shell: bash + run: | + # Specific oneapi position on arc ut test machines + if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then + source /opt/intel/oneapi/setvars.sh + elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then + source /home/arda/intel/oneapi/setvars.sh + fi + bash python/llm/test/run-llm-install-tests.sh + + - name: Download LLMs and datasets + shell: bash + run: | + if [ ! -d $LLAMA2_7B_ORIGIN_PATH ]; then + echo "Directory $LLAMA2_7B_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Llama-2-7b-chat-hf -P $ORIGIN_DIR + fi + if [ ! -d $CHATGLM2_6B_ORIGIN_PATH ]; then + echo "Directory $CHATGLM2_6B_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/chatglm2-6b -P $ORIGIN_DIR + fi + if [ ! -d $FALCON_7B_ORIGIN_PATH ]; then + echo "Directory $FALCON_7B_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/falcon-7b-instruct-with-patch -P $ORIGIN_DIR + fi + if [ ! -d $MPT_7B_ORIGIN_PATH ]; then + echo "Directory $MPT_7B_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/mpt-7b-chat -P $ORIGIN_DIR + fi + if [ ! -d $WHISPER_TINY_ORIGIN_PATH ]; then + echo "Directory $WHISPER_TINY_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/whisper-tiny -P $ORIGIN_DIR + fi + if [ ! -d $DATASET_DIR ]; then + mkdir -p $DATASET_DIR + fi + if [ ! -d $ABIRATE_ENGLISH_QUOTES_PATH ]; then + echo "Directory $ABIRATE_ENGLISH_QUOTES_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/abirate_english_quotes -P $DATASET_DIR + fi + if [ ! -d $SPEECH_DATASET_PATH ]; then + echo "Directory $SPEECH_DATASET_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/librispeech_asr_dummy -P $DATASET_DIR + fi + if [ ! -d $MISTRAL_7B_INSTRUCT_V0_1_ORIGIN_PATH ]; then + echo "Directory $MISTRAL_7B_INSTRUCT_V0_1_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Mistral-7B-Instruct-v0.1 -P $ORIGIN_DIR + fi + if [ ! -d $QWEN_7B_ORIGIN_PATH ]; then + echo "Directory $QWEN_7B_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Qwen-7B-Chat -P $ORIGIN_DIR + fi + if [ ! -d $BAICHUAN2_7B_ORIGIN_PATH ]; then + echo "Directory $BAICHUAN2_7B_ORIGIN_PATH not found. Downloading from FTP server..." + wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Baichuan2-7B-Chat -P $ORIGIN_DIR + fi + + - name: Run LLM inference test + shell: bash + run: | + # Specific oneapi position on arc ut test machines + if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then + source /opt/intel/oneapi/setvars.sh + elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then + source /home/arda/intel/oneapi/setvars.sh + fi + python -m pip install datasets librosa soundfile einops tiktoken transformers_stream_generator + bash python/llm/test/run-llm-inference-tests-gpu.sh + python -m pip install transformers==4.34.0 + bash python/llm/test/run-llm-inference-tests-gpu-434.sh + + - name: Run LLM example tests + shell: bash + run: | + python -m pip uninstall datasets -y + python -m pip install transformers==4.34.0 datasets peft==0.5.0 accelerate==0.23.0 + python -m pip install bitsandbytes scipy + # Specific oneapi position on arc ut test machines + if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then + source /opt/intel/oneapi/setvars.sh + elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then + source /home/arda/intel/oneapi/setvars.sh + fi + bash python/llm/test/run-llm-example-tests-gpu.sh + + - name: Run LLM langchain GPU test + shell: bash + run: | + pip install -U langchain==0.0.184 + pip install -U chromadb==0.3.25 + pip install -U pandas==2.0.3 + # Specific oneapi position on arc ut test machines + if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then + source /opt/intel/oneapi/setvars.sh + elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then + source /home/arda/intel/oneapi/setvars.sh + fi + bash python/llm/test/run-llm-langchain-tests-gpu.sh + + - name: Run LLM llamaindex GPU test + shell: bash + run: | + pip install llama-index-readers-file llama-index-vector-stores-postgres llama-index-embeddings-huggingface + # Specific oneapi position on arc ut test machines + if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then + pip install --pre --upgrade bigdl-llm[xpu] -f https://developer.intel.com/ipex-whl-stable-xpu + source /opt/intel/oneapi/setvars.sh + elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then + pip install --pre --upgrade bigdl-llm[xpu_2.0] -f https://developer.intel.com/ipex-whl-stable-xpu + source /home/arda/intel/oneapi/setvars.sh + fi + bash python/llm/test/run-llm-llamaindex-tests-gpu.sh \ No newline at end of file