Merge pull request #325 from abstractqqq/update_ci_to_support_lts_cpu

Improve CI

.github/workflows/CI.yml

@@ -49,12 +49,13 @@ jobs:
           args: >
             --manifest-path Cargo.toml
             --out dist
+          maturin-version: 1.7.4
 
       - name: Test sdist
         run: |
-          pip install --force-reinstall --verbose dist/*.tar.gz
-          pip install typing_extensions
           pip install -r requirements.txt
+          pip install typing_extensions
+          pip install --force-reinstall --verbose dist/*.tar.gz
           python -c 'import polars_ds as pds'
           python -c 'from polars_ds import linear_models'
           python -c 'from polars_ds.spatial import *'
@@ -67,24 +68,6 @@ jobs:
           name: sdist-${{ matrix.package }}
           path: dist/*.tar.gz
 
-      - name: Test Correctness
-        run: | 
-          python -m pip install --upgrade pip
-          pip install pytest
-          pip install .
-          pip install -r tests/requirements-test.txt
-          pytest tests/test_*
-
-      - name: Test Notebooks
-        run: | 
-          python -m pip install --upgrade pip
-          pip install jupyter ipython ipykernel nbconvert
-          pip install -r tests/requirements-test.txt
-          jupyter execute examples/basics.ipynb
-          jupyter execute examples/pipeline.ipynb
-          jupyter execute examples/eda.ipynb
-          jupyter execute examples/sample_and_split.ipynb
-
 
   build-wheels:
     runs-on: ${{ matrix.os }}
@@ -112,27 +95,52 @@ jobs:
       - name: Determine CPU features for x86-64
         id: features
         if: matrix.architecture == 'x86-64'
-        env:
-          IS_MACOS: ${{ matrix.os == 'macos-13' }}
+
+        # env:
+        #   IS_LTS_CPU: ${{ matrix.package == 'polars_ds_lts_cpu' }}
+
+        # if [[ "$IS_LTS_CPU" = true ]]; then
+        #   FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b
+        #   CC_FEATURES="-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16"
+        # else
+        # fi
         run: |
-          if [[ "$IS_MACOS" = true ]]; then
-            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma,+pclmulqdq
-          else
-            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq
-          fi
+          TUNE_CPU=skylake
+          FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq,+movbe
+          CC_FEATURES="-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mpclmul -mmovbe"
+          
           echo "features=$FEATURES" >> $GITHUB_OUTPUT
+          echo "tune_cpu=$TUNE_CPU" >> $GITHUB_OUTPUT
+          echo "cc_features=$CC_FEATURES" >> $GITHUB_OUTPUT
 
       - name: Set RUSTFLAGS for x86-64
         if: matrix.architecture == 'x86-64'
         env:
           FEATURES: ${{ steps.features.outputs.features }}
-        run: echo "RUSTFLAGS=-C target-feature=${{ steps.features.outputs.features }}" >> $GITHUB_ENV
+          TUNE_CPU: ${{ steps.features.outputs.tune_cpu }}
+          CC_FEATURES: ${{ steps.features.outputs.cc_features }}
+          # CFG: ${{ matrix.package == 'polars_ds_lts_cpu' && '--cfg allocator="default"' || '' }}
+          # add $CFG
+        run: |
+          if [[ -z "$TUNE_CPU" ]]; then
+            echo "RUSTFLAGS=-C target-feature=$FEATURES" >> $GITHUB_ENV
+            echo "CFLAGS=$CC_FEATURES" >> $GITHUB_ENV
+          else
+            echo "RUSTFLAGS=-C target-feature=$FEATURES -Z tune-cpu=$TUNE_CPU" >> $GITHUB_ENV
+            echo "CFLAGS=$CC_FEATURES -mtune=$TUNE_CPU" >> $GITHUB_ENV
+          fi
 
       - name: Set Rust target for aarch64
         if: matrix.architecture == 'aarch64'
         id: target
         run: |
-          TARGET=${{ matrix.os == 'macos-13' && 'aarch64-apple-darwin' || 'aarch64-unknown-linux-gnu'}}
+          TARGET=$(
+            if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+              echo "aarch64-apple-darwin";
+            else
+              echo "aarch64-unknown-linux-gnu";
+            fi
+          )
           echo "target=$TARGET" >> $GITHUB_OUTPUT
 
       - name: Set jemalloc for aarch64 Linux
@@ -163,7 +171,6 @@ jobs:
           python -c 'from polars_ds import linear_models'
           python -c 'from polars_ds.spatial import *'
           python -c 'from polars_ds.sample_and_split import *'
-          python -c 'from polars_ds.exprs.ts_features import *'
 
       - name: Upload wheel
         uses: actions/upload-artifact@v4

.github/workflows/python-tests.yml

@@ -0,0 +1,79 @@
+name: Test Python
+
+on:
+  pull_request:
+    paths:
+      - Cargo.lock
+      - python/**
+      - src/**
+      - .github/workflows/python-tests.yml
+  push:
+    branches:
+      - main
+    paths:
+      - Cargo.lock
+      - python/**
+      - src/**
+      - .github/workflows/python-tests.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUSTFLAGS: -C debuginfo=0  # Do not produce debug symbols to keep memory usage down
+  RUST_BACKTRACE: 1
+  PYTHONUTF8: 1
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  test-python:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.9', '3.12', '3.13']
+        # include:
+        #   - os: windows-latest
+        #     python-version: '3.13'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Create virtual environment
+        env:
+          BIN: ${{ matrix.os == 'windows-latest' && 'Scripts' || 'bin' }}
+        run: |
+          python -m venv .venv
+          echo "$GITHUB_WORKSPACE/.venv/$BIN" >> $GITHUB_PATH
+          echo "VIRTUAL_ENV=$GITHUB_WORKSPACE/.venv" >> $GITHUB_ENV
+      
+      - name: Install package
+        run: | 
+          pip install -r requirements.txt
+          maturin develop --release
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install jupyter ipython ipykernel nbconvert pytest
+          pip install -r tests/requirements-test.txt
+
+      - name: Test Notebooks
+        run: | 
+          jupyter execute examples/basics.ipynb
+          jupyter execute examples/pipeline.ipynb
+          jupyter execute examples/eda.ipynb
+          jupyter execute examples/sample_and_split.ipynb
+
+      - name: Test Pytests
+        run: pytest tests/test_*

README.md

@@ -252,7 +252,9 @@ If your code already executes under 1s and you only use your code in non-product
 
 **Currently in Beta. Feel free to submit feature requests in the issues section of the repo. This library will only depend on python Polars (for most of its core) and will try to be as stable as possible for polars>=1. Exceptions will be made when Polars's update forces changes in the plugins.**
 
-This package is not tested with Polars streaming mode and is not designed to work with data so big that has to be streamed.
+This package is not tested with Polars streaming mode and is not designed to work with data so big that has to be streamed. By the same token, Polars large index version is not intentionally supported at this point.
+
+On the other hand, for polars-lts-cpu users, there will be support.
 
 # Credits
 

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["maturin>=1.3.2"]
+requires = ["maturin>=1.7.4"]
 build-backend = "maturin"
 
 [project]
@@ -24,7 +24,6 @@ dependencies = [
 keywords = ["polars-extension", "scientific-computing", "data-science"]
 
 [project.optional-dependencies]
-# "plotly >= 5.5"
 plot = ["great-tables>=0.9", "graphviz>=0.20", "altair >= 5.4.0", "vegafusion[embed]"]
 models = ["numpy>=1.16"]
 compat = ["numpy>=1.16"]

python/polars_ds/exprs/expr_iter.py

@@ -65,12 +65,12 @@ def combinations(source: str | pl.Expr, k: int, unique: bool = False) -> pl.Expr
     │ b        ┆ [[4, 5]]                 │
     └──────────┴──────────────────────────┘
     """
+    s = str_to_expr(source).unique().sort() if unique else str_to_expr(source)
     return pl_plugin(
         symbol="pl_combinations",
-        args=[str_to_expr(source)],
+        args=[s],
         changes_length=True,
         kwargs={
-            "unique": unique,
             "k": k,
         },
     )

src/num_ext/isotonic_regression.rs

@@ -7,6 +7,9 @@ use serde::Deserialize;
 // https://github.com/scipy/scipy/blob/v1.14.1/scipy/optimize/_pava/pava_pybind.cpp
 // https://www.jstatsoft.org/article/view/v102c01
 
+// The code here has to be compiled in --release
+// Otherwise, a mysterious error will occur. Thank you compiler!
+
 #[derive(Deserialize, Debug)]
 pub(crate) struct IsotonicRegKwargs {
     pub(crate) has_weights: bool,
@@ -52,7 +55,6 @@ fn isotonic_regression(x: &mut [f64], w: &mut [f64], r: &mut [usize]) {
         w[b] = wb;
 
         r[b + 1] = i + 1;
-
         i += 1;
     }
 
@@ -119,7 +121,7 @@ fn pl_isotonic_regression(inputs: &[Series], kwargs: IsotonicRegKwargs) -> Polar
         y.reverse();
     }
 
-    let mut r = vec![y.len() - 1; y.len() + 1];
+    let mut r = vec![0; y.len() + 1];
     isotonic_regression(&mut y, &mut w, &mut r);
     if !increasing {
         y.reverse();

src/num_ext/iterations.rs

@@ -5,7 +5,6 @@ use serde::Deserialize;
 
 #[derive(Deserialize, Debug)]
 pub(crate) struct CombinationKwargs {
-    pub(crate) unique: bool,
     pub(crate) k: usize,
 }
 
@@ -70,11 +69,7 @@ fn get_combinations_str(
 #[polars_expr(output_type_func=combination_output)]
 fn pl_combinations(inputs: &[Series], kwargs: CombinationKwargs) -> PolarsResult<Series> {
 
-    let s = if kwargs.unique {
-        inputs[0].unique()?
-    } else {
-        inputs[0].clone()
-    };
+    let s = &inputs[0];
     let k = kwargs.k;
 
     if s.len() < k {