feat(machine-learning): add sglang

hongbo-miao · Jan 3, 2025 · 9c716c7 · 9c716c7
1 parent 530fc07
commit 9c716c7
Show file tree

Hide file tree

Showing 10 changed files with 2,115 additions and 526 deletions.
diff --git a/.github/workflows/.static-type-check.yml b/.github/workflows/.static-type-check.yml
@@ -173,6 +173,7 @@ jobs:
           uv run poe static-type-check-python --package=machine-learning.hm-mlflow.experiments.classify-mnist
           uv run poe static-type-check-python --package=machine-learning.hm-mlflow.experiments.predict-diabetes
           uv run poe static-type-check-python --package=machine-learning.hm-rasa
+          uv run poe static-type-check-python --package=machine-learning.hm-sglang
           uv run poe static-type-check-python --package=machine-learning.hm-streamlit.applications.live-line-chart
           uv run poe static-type-check-python --package=machine-learning.hm-streamlit.applications.map
           uv run poe static-type-check-python --package=machine-learning.hm-supervision.detect-objects

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -93,6 +93,7 @@ jobs:
       hm-ray-calculate: ${{ steps.filter.outputs.hm-ray-calculate }}
       hm-ray-process-flight-data: ${{ steps.filter.outputs.hm-ray-process-flight-data }}
       hm-serial: ${{ steps.filter.outputs.hm-serial }}
+      hm-sglang: ${{ steps.filter.outputs.hm-sglang }}
       hm-skypilot: ${{ steps.filter.outputs.hm-skypilot }}
       hm-spark-analyze-coffee-customers: ${{ steps.filter.outputs.hm-spark-analyze-coffee-customers }}
       hm-spark-find-retired-people-python: ${{ steps.filter.outputs.hm-spark-find-retired-people-python }}
@@ -358,6 +359,9 @@ jobs:
             hm-serial:
               - '.github/workflows/test.yml'
               - 'embedded/hm-serial/**'
+            hm-sglang:
+              - '.github/workflows/test.yml'
+              - 'machine-learning/hm-sglang/**'
             hm-skypilot:
               - '.github/workflows/test.yml'
               - 'cloud-computing/hm-skypilot/**'
@@ -1603,106 +1607,6 @@ jobs:
         with:
           directory: data-storage/hm-protobuf
 
-  autogluon-test:
-    name: AutoGluon | Test
-    needs: detect-changes
-    if: ${{ needs.detect-changes.outputs.hm-autogluon == 'true' }}
-    runs-on: ubuntu-24.04
-    environment: test
-    timeout-minutes: 10
-    steps:
-      - name: Checkout
-        uses: actions/[email protected]
-      - name: Install uv
-        uses: astral-sh/[email protected]
-        with:
-          version: 0.5.13
-          enable-cache: true
-          cache-dependency-glob: machine-learning/hm-autogluon/uv.lock
-      - name: Set up Python
-        uses: actions/[email protected]
-        with:
-          python-version-file: machine-learning/hm-autogluon/pyproject.toml
-      - name: Install dependencies
-        working-directory: machine-learning/hm-autogluon
-        run: |
-          uv sync --dev
-      - name: Test
-        working-directory: machine-learning/hm-autogluon
-        run: |
-          uv run poe test-coverage
-      - name: Upload coverage to Codecov
-        uses: codecov/[email protected]
-        with:
-          directory: machine-learning/hm-autogluon
-
-  dali-test:
-    name: DALI | Test
-    needs: detect-changes
-    if: ${{ false }}
-    # if: ${{ needs.detect-changes.outputs.dali == 'true' }}
-    runs-on: ubuntu-24.04
-    environment: test
-    timeout-minutes: 10
-    steps:
-      - name: Checkout
-        uses: actions/[email protected]
-      - name: Install uv
-        uses: astral-sh/[email protected]
-        with:
-          version: 0.5.13
-          enable-cache: true
-          cache-dependency-glob: machine-learning/dali/uv.lock
-      - name: Set up Python
-        uses: actions/[email protected]
-        with:
-          python-version-file: machine-learning/dali/pyproject.toml
-      - name: Install dependencies
-        working-directory: machine-learning/dali
-        run: |
-          uv sync --dev
-      - name: Test
-        working-directory: machine-learning/dali
-        run: |
-          uv run poe test-coverage
-      - name: Upload coverage to Codecov
-        uses: codecov/[email protected]
-        with:
-          directory: machine-learning/dali
-
-  feature-store-test:
-    name: Feature Store | Test
-    needs: detect-changes
-    if: ${{ needs.detect-changes.outputs.feature-store == 'true' }}
-    runs-on: ubuntu-24.04
-    environment: test
-    timeout-minutes: 10
-    steps:
-      - name: Checkout
-        uses: actions/[email protected]
-      - name: Install uv
-        uses: astral-sh/[email protected]
-        with:
-          version: 0.5.13
-          enable-cache: true
-          cache-dependency-glob: machine-learning/feature-store/uv.lock
-      - name: Set up Python
-        uses: actions/[email protected]
-        with:
-          python-version-file: machine-learning/feature-store/pyproject.toml
-      - name: Install dependencies
-        working-directory: machine-learning/feature-store
-        run: |
-          uv sync --dev
-      - name: Test
-        working-directory: machine-learning/feature-store
-        run: |
-          uv run poe test-coverage
-      - name: Upload coverage to Codecov
-        uses: codecov/[email protected]
-        with:
-          directory: machine-learning/feature-store
-
   cupy-test:
     name: CuPy | Test
     needs: detect-changes
@@ -2132,6 +2036,139 @@ jobs:
         with:
           directory: machine-learning/hm-flax
 
+  autogluon-test:
+    name: AutoGluon | Test
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.hm-autogluon == 'true' }}
+    runs-on: ubuntu-24.04
+    environment: test
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Install uv
+        uses: astral-sh/[email protected]
+        with:
+          version: 0.5.13
+          enable-cache: true
+          cache-dependency-glob: machine-learning/hm-autogluon/uv.lock
+      - name: Set up Python
+        uses: actions/[email protected]
+        with:
+          python-version-file: machine-learning/hm-autogluon/pyproject.toml
+      - name: Install dependencies
+        working-directory: machine-learning/hm-autogluon
+        run: |
+          uv sync --dev
+      - name: Test
+        working-directory: machine-learning/hm-autogluon
+        run: |
+          uv run poe test-coverage
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
+        with:
+          directory: machine-learning/hm-autogluon
+
+  dali-test:
+    name: DALI | Test
+    needs: detect-changes
+    if: ${{ false }}
+    # if: ${{ needs.detect-changes.outputs.dali == 'true' }}
+    runs-on: ubuntu-24.04
+    environment: test
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Install uv
+        uses: astral-sh/[email protected]
+        with:
+          version: 0.5.13
+          enable-cache: true
+          cache-dependency-glob: machine-learning/dali/uv.lock
+      - name: Set up Python
+        uses: actions/[email protected]
+        with:
+          python-version-file: machine-learning/dali/pyproject.toml
+      - name: Install dependencies
+        working-directory: machine-learning/dali
+        run: |
+          uv sync --dev
+      - name: Test
+        working-directory: machine-learning/dali
+        run: |
+          uv run poe test-coverage
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
+        with:
+          directory: machine-learning/dali
+
+  feature-store-test:
+    name: Feature Store | Test
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.feature-store == 'true' }}
+    runs-on: ubuntu-24.04
+    environment: test
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Install uv
+        uses: astral-sh/[email protected]
+        with:
+          version: 0.5.13
+          enable-cache: true
+          cache-dependency-glob: machine-learning/feature-store/uv.lock
+      - name: Set up Python
+        uses: actions/[email protected]
+        with:
+          python-version-file: machine-learning/feature-store/pyproject.toml
+      - name: Install dependencies
+        working-directory: machine-learning/feature-store
+        run: |
+          uv sync --dev
+      - name: Test
+        working-directory: machine-learning/feature-store
+        run: |
+          uv run poe test-coverage
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
+        with:
+          directory: machine-learning/feature-store
+
+  sglang-test:
+    name: SGLang | Test
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.sglang == 'true' }}
+    runs-on: ubuntu-24.04
+    environment: test
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Install uv
+        uses: astral-sh/[email protected]
+        with:
+          version: 0.5.13
+          enable-cache: true
+          cache-dependency-glob: machine-learning/sglang/uv.lock
+      - name: Set up Python
+        uses: actions/[email protected]
+        with:
+          python-version-file: machine-learning/sglang/pyproject.toml
+      - name: Install dependencies
+        working-directory: machine-learning/sglang
+        run: |
+          uv sync --dev
+      - name: Test
+        working-directory: machine-learning/sglang
+        run: |
+          uv run poe test-coverage
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]
+        with:
+          directory: machine-learning/sglang
+
   kubeflow-calculate-test:
     name: Kubeflow (calculate) | Test
     needs: detect-changes

diff --git a/.mergify.yml b/.mergify.yml
@@ -278,15 +278,6 @@ pull_request_rules:
       - or:
           - check-success=Protobuf | Test
           - check-skipped=Protobuf | Test
-      - or:
-          - check-success=AutoGluon | Test
-          - check-skipped=AutoGluon | Test
-      # - or:
-      #     - check-success=DALI | Test
-      #     - check-skipped=DALI | Test
-      - or:
-          - check-success=Feature Store | Test
-          - check-skipped=Feature Store | Test
       - or:
           - check-success=CuPy | Test
           - check-skipped=CuPy | Test
@@ -326,6 +317,18 @@ pull_request_rules:
       - or:
           - check-success=Flax | Test
           - check-skipped=Flax | Test
+      - or:
+          - check-success=AutoGluon | Test
+          - check-skipped=AutoGluon | Test
+      # - or:
+      #     - check-success=DALI | Test
+      #     - check-skipped=DALI | Test
+      - or:
+          - check-success=Feature Store | Test
+          - check-skipped=Feature Store | Test
+      - or:
+          - check-success=SGLang | Test
+          - check-skipped=SGLang | Test
       - or:
           - check-success=Kubeflow (calculate) | Test
           - check-skipped=Kubeflow (calculate) | Test

diff --git a/Makefile b/Makefile
@@ -382,6 +382,7 @@ static-type-check-python:
 	uv run poe static-type-check-python --package=machine-learning.hm-mlflow.experiments.classify-mnist
 	uv run poe static-type-check-python --package=machine-learning.hm-mlflow.experiments.predict-diabetes
 	uv run poe static-type-check-python --package=machine-learning.hm-rasa
+	uv run poe static-type-check-python --package=machine-learning.hm-sglang
 	uv run poe static-type-check-python --package=machine-learning.hm-streamlit.applications.live-line-chart
 	uv run poe static-type-check-python --package=machine-learning.hm-streamlit.applications.map
 	uv run poe static-type-check-python --package=machine-learning.hm-supervision.detect-objects

diff --git a/README.md b/README.md
@@ -443,6 +443,7 @@ The diagram illustrates the repository's architecture, which is considered overl
 - **MinerU** - Document data extraction
 - **Docling** - Document data extraction
 - **pypdf** - PDF manipulation
+- **SGLang** - Large language model (LLM) and visual language model (VLM) serving framework
 - **LiteLLM** - LLM gateway
 - **Open WebUI** - AI chat interface
   - **Open WebUI Pipelines** - OpenAI API plugin framework

diff --git a/machine-learning/hm-sglang/Makefile b/machine-learning/hm-sglang/Makefile
@@ -5,6 +5,11 @@ uv-update-lock-file:
 uv-install-dependencies:
 	uv sync --dev
 
+uv-run-download-model:
+	uv run poe download-model
+uv-run-launch-server:
+	uv run poe launch-server
+
 uv-run-dev:
 	uv run poe dev
 uv-run-test:

diff --git a/machine-learning/hm-sglang/pyproject.toml b/machine-learning/hm-sglang/pyproject.toml
@@ -3,11 +3,13 @@ name = "hm-sglang"
 version = "1.0.0"
 requires-python = "~=3.12.0"
 dependencies = [
+  "flashinfer",
   "sglang[all]==0.4.1.post3",
 ]
 
 [dependency-groups]
 dev = [
+  "huggingface-hub==0.27.0",
   "poethepoet==0.32.0",
   "pytest==8.3.4",
   "pytest-cov==6.0.0",
@@ -16,7 +18,17 @@ dev = [
 [tool.uv]
 package = false
 
+[tool.uv.sources]
+flashinfer = { index = "flashinfer" }
+
+[[tool.uv.index]]
+name = "flashinfer"
+url = "https://flashinfer.ai/whl/cu124/torch2.4"
+
 [tool.poe.tasks]
-dev = "streamlit run src/main.py"
+download-model = "python src/download_model.py"
+launch-server = "python -m sglang.launch_server --model-path=Qwen/Qwen2.5-0.5B-Instruct --port=19863"
+
+dev = "python src/main.py"
 test = "pytest --verbose --verbose"
 test-coverage = "pytest --cov=. --cov-report=xml"
diff --git a/machine-learning/hm-sglang/src/download_model.py b/machine-learning/hm-sglang/src/download_model.py
@@ -0,0 +1,18 @@
+import logging
+
+from huggingface_hub import snapshot_download
+
+logger = logging.getLogger(__name__)
+
+
+def download_model() -> None:
+    logger.info("Downloading model...")
+    model_path = snapshot_download("Qwen/Qwen2.5-0.5B-Instruct")
+    logger.info(f"Model downloaded successfully to: {model_path}")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+    )
+    download_model()