Skip to content

Commit

Permalink
Merge branch 'main' into diffusion
Browse files Browse the repository at this point in the history
  • Loading branch information
ivy-lv11 authored Jun 11, 2024
2 parents 22245fd + 70b17c8 commit aeb287f
Show file tree
Hide file tree
Showing 191 changed files with 4,374 additions and 2,955 deletions.
105 changes: 105 additions & 0 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
# push:
# branches: [ "main" ]
schedule:
- cron: '24 14 * * 0'
workflow_dispatch:

jobs:
analyze:
name: Analyze (${{ matrix.language }})
# Runner size impacts CodeQL analysis time. To learn more, please see:
# - https://gh.io/recommended-hardware-resources-for-running-codeql
# - https://gh.io/supported-runners-and-hardware-resources
# - https://gh.io/using-larger-runners (GitHub.com only)
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
permissions:
# required for all workflows
security-events: write

# required to fetch internal or private CodeQL packs
packages: read

# only required for workflows in private repositories
actions: read
contents: read

strategy:
fail-fast: false
matrix:
include:
- language: python
build-mode: none
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
# Use `c-cpp` to analyze code written in C, C++ or both
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
steps:
- name: Checkout repository
uses: actions/checkout@v4

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.

# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality

# If the analyze step fails for one of the languages you are analyzing with
# "We were unable to automatically build your code", modify the matrix above
# to set the build mode to "manual" for that language. Then modify this step
# to build your code.
# ℹ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
- if: matrix.build-mode == 'manual'
shell: bash
run: |
echo 'If you are using a "manual" build mode for one or more of the' \
'languages you are analyzing, replace this with the commands to build' \
'your code, for example:'
echo ' make bootstrap'
echo ' make release'
exit 1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"

- name: Generate Security Report
uses: rsdmike/[email protected]
with:
template: report
token: ${{ secrets.SECURITY_TOKEN }}

- name: GitHub Upload Release Artifacts
uses: actions/upload-artifact@v2
with:
name: report
path: |
./report.pdf
102 changes: 1 addition & 101 deletions .github/workflows/llm-binary-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,6 @@ jobs:
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
yum install -y gcc-toolset-11 cmake git
conda remove -n python39 --all -y
conda create -n python39 python=3.9 -y
conda remove -n python310 --all -y
conda create -n python310 python=3.10 -y
conda remove -n python311 --all -y
conda create -n python311 python=3.11 -y
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
with:
repository: "intel-analytics/llm.cpp"
Expand Down Expand Up @@ -109,42 +103,6 @@ jobs:
mv build/libstarcoder-api.so release/libstarcoder-api.so
mv build/quantize-starcoder release/quantize-starcoder
mv build/libstarcoder.so release/libstarcoder_avxvnni.so
- name: Build Chatglm
shell: bash
run: |
source activate python39 || conda activate python39
cd src/chatglm
scl enable gcc-toolset-11 "cmake -B build"
scl enable gcc-toolset-11 "cmake --build build --config Release -j"
- name: Move Chatglm binaries
shell: bash
run: |
mv src/chatglm/build/main release/main-chatglm_vnni
mv src/chatglm/build/_C.cpython-39-x86_64-linux-gnu.so release/chatglm_C.cpython-39-x86_64-linux-gnu.so
- name: Build Chatglm Py310
shell: bash
run: |
source activate python310 || conda activate python310
cd src/chatglm
rm -r build
scl enable gcc-toolset-11 "cmake -B build"
scl enable gcc-toolset-11 "cmake --build build --config Release -j"
- name: Move Chatglm binaries Py310
shell: bash
run: |
mv src/chatglm/build/_C.cpython-310-x86_64-linux-gnu.so release/chatglm_C.cpython-310-x86_64-linux-gnu.so
- name: Build Chatglm Py311
shell: bash
run: |
source activate python311 || conda activate python311
cd src/chatglm
rm -r build
scl enable gcc-toolset-11 "cmake -B build"
scl enable gcc-toolset-11 "cmake --build build --config Release -j"
- name: Move Chatglm binaries Py311
shell: bash
run: |
mv src/chatglm/build/_C.cpython-311-x86_64-linux-gnu.so release/chatglm_C.cpython-311-x86_64-linux-gnu.so
- name: Archive build files
uses: actions/upload-artifact@v3
with:
Expand All @@ -155,9 +113,6 @@ jobs:
shell: bash
run: |
make clean
conda remove -n python39 --all -y
conda remove -n python310 --all -y
conda remove -n python311 --all -y
check-linux-avx512-artifact:
if: ${{contains(inputs.platform, 'Linux')}}
Expand Down Expand Up @@ -255,6 +210,7 @@ jobs:
path: |
avx_release
- name: Clean up test environment
if: ${{ always() }}
shell: bash
run: |
make clean
Expand Down Expand Up @@ -286,8 +242,6 @@ jobs:
export http_proxy=${HTTP_PROXY}
export https_proxy=${HTTPS_PROXY}
yum install -y gcc-toolset-11 cmake git
conda remove -n python39 --all -y
conda create -n python39 python=3.9 -y
- uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
with:
repository: "intel-analytics/llm.cpp"
Expand All @@ -299,11 +253,6 @@ jobs:
run: |
scl enable gcc-toolset-11 "cmake -DONLYAVX=OFF -DONLYAVX2=OFF -B build"
scl enable gcc-toolset-11 "cmake --build build --config Release -j"
# build chatglm
source activate python39 || conda activate python39
cd src/chatglm
scl enable gcc-toolset-11 "cmake -B build"
scl enable gcc-toolset-11 "cmake --build build --config Release -j"
- name: Move amx release binary
shell: bash
run: |
Expand All @@ -316,9 +265,6 @@ jobs:
mv build/libgptneox.so amx_release/libgptneox_amx.so
mv build/quantize-starcoder amx_release/quantize-starcoder_amx
mv build/libstarcoder.so amx_release/libstarcoder_amx.so
# chatglm binary files
mv src/chatglm/build/main amx_release/main-chatglm_amx
# mv src/chatglm/build/_C.cpython-39-x86_64-linux-gnu.so amx_release/chatglm_C.cpython-39-x86_64-linux-gnu.so
- name: Archive amx build files
uses: actions/upload-artifact@v3
with:
Expand All @@ -329,7 +275,6 @@ jobs:
shell: bash
run: |
make clean
conda remove -n python39 --all -y
check-windows-avx2-artifact:
if: ${{contains(inputs.platform, 'Windows')}}
Expand Down Expand Up @@ -393,10 +338,6 @@ jobs:
needs: check-windows-avx-vnni-artifact
if: needs.check-windows-avx-vnni-artifact.outputs.if-exists == 'false'
steps:
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.9"
- name: Set access token
run: |
echo "github_access_token=$env:GITHUB_ACCESS_TOKEN" >> $env:GITHUB_ENV
Expand Down Expand Up @@ -438,47 +379,6 @@ jobs:
# mv build/Release/main-starcoder.exe release/main-starcoder_vnni.exe
mv build/Release/quantize-starcoder.exe release/quantize-starcoder_vnni.exe
mv build/Release/starcoder.dll release/libstarcoder_vnni.dll
- name: Build Chatglm
shell: powershell
run: |
cd src/chatglm
cmake -DAVXVNNI=ON -B build
cmake --build build --config Release -j
- name: Move Chatglm binaries
shell: powershell
run: |
mv src/chatglm/build/Release/main.exe release/main-chatglm_vnni.exe
mv src/chatglm/build/Release/_C.cp39-win_amd64.pyd release/chatglm_C.cp39-win_amd64.pyd
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Build Chatglm Py310
shell: powershell
run: |
cd src/chatglm
rm -r build
cmake -DAVXVNNI=ON -B build
cmake --build build --config Release -j
- name: Move Chatglm binaries Py310
shell: powershell
run: |
mv src/chatglm/build/Release/_C.cp310-win_amd64.pyd release/chatglm_C.cp310-win_amd64.pyd
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Build Chatglm Py311
shell: powershell
run: |
cd src/chatglm
rm -r build
cmake -DAVXVNNI=ON -B build
cmake --build build --config Release -j
- name: Move Chatglm binaries Py311
shell: powershell
run: |
mv src/chatglm/build/Release/_C.cp311-win_amd64.pyd release/chatglm_C.cp311-win_amd64.pyd
- name: Archive build files
uses: actions/upload-artifact@v3
with:
Expand Down
Loading

0 comments on commit aeb287f

Please sign in to comment.