feat(query): add python udf script runtime

.github/actions/build_linux/action.yml

@@ -30,7 +30,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,PYO3_PYTHON,LD_LIBRARY_PATH,PYO3_CROSS_PYTHON_VERSION
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')
@@ -61,6 +61,9 @@ runs:
         echo "RUSTFLAGS=${flags} -C link-arg=-Wl,--compress-debug-sections=zlib" >> $GITHUB_ENV
         target=${{ inputs.target }}
         echo "BUILD_ARCH=${target/-unknown-linux-*}" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=/usr/local/lib" >> $GITHUB_ENV
+        echo "PYO3_PYTHON=python3.12" >> $GITHUB_ENV
+        echo "PYO3_CROSS_PYTHON_VERSION=/3.12" >> $GITHUB_ENV
 
     # build all binaries for debug
     - name: Build Debug

.github/actions/build_linux_sanitizer/action.yml

@@ -19,7 +19,7 @@ runs:
       uses: ./.github/actions/setup_build_tool
       with:
         target: ${{ inputs.target }}
-        bypass_env_vars: RUSTFLAGS,RUST_LOG
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,PYO3_PYTHON,LD_LIBRARY_PATH
 
     - name: Cross setup
       if: startsWith(inputs.target, 'aarch64-')

.github/actions/check/action.yml

@@ -10,7 +10,7 @@ runs:
     - name: Setup Build Tool
       uses: ./.github/actions/setup_build_tool
       with:
-        bypass_env_vars: RUSTFLAGS,RUST_LOG,GITHUB_TOKEN
+        bypass_env_vars: RUSTFLAGS,RUST_LOG,GITHUB_TOKEN,PYO3_PYTHON,LD_LIBRARY_PATH
 
     - name: Check Apache License Header
       uses: korandoru/hawkeye@v2

.github/actions/setup_build_tool/action.yml

@@ -8,7 +8,7 @@ inputs:
   bypass_env_vars:
     description: "Environment variables bypass to docker container"
     required: false
-    default: RUSTFLAGS,RUST_LOG
+    default: RUSTFLAGS,RUST_LOG,PYO3_PYTHON,LD_LIBRARY_PATH
 runs:
   using: "composite"
   steps:
@@ -28,6 +28,9 @@ runs:
         cat <<EOF >$BIN_LOCAL/build-tool
         #!/bin/bash
         script_name=\$(basename "\$0")
+        export PYO3_PYTHON=python3.12
+        export LD_LIBRARY_PATH=/usr/local/lib
+        export PYO3_CROSS_PYTHON_VERSION=3.12
         export INTERACTIVE=false
         export TARGET=${{ inputs.target }}
         export CARGO_INCREMENTAL=0

.github/actions/test_unit/action.yml

@@ -7,7 +7,7 @@ runs:
     - name: Setup Build Tool
       uses: ./.github/actions/setup_build_tool
       with:
-        bypass_env_vars: RUSTFLAGS,RUSTDOCFLAGS,RUST_TEST_THREADS,RUST_LOG,RUST_BACKTRACE
+        bypass_env_vars: RUSTFLAGS,RUSTDOCFLAGS,RUST_TEST_THREADS,RUST_LOG,RUST_BACKTRACE,PYO3_PYTHON,LD_LIBRARY_PATH
 
     - shell: bash
       run: |

.github/workflows/bindings.python.yml

@@ -1,18 +1,20 @@
 name: Bindings Python
 
 on:
-  pull_request:
-    branches:
-      - main
-    paths:
-      - "src/**"
-      - ".github/workflows/bindings.python.yml"
-  workflow_call:
-    inputs:
-      tag:
-        description: Tag to release
-        required: true
-        type: string
+  ## uncomment it when bendpy is enabled
+  workflow_dispatch:
+  # pull_request:
+  #   branches:
+  #     - main
+  #   paths:
+  #     - "src/**"
+  #     - ".github/workflows/bindings.python.yml"
+  # workflow_call:
+  #   inputs:
+  #     tag:
+  #       description: Tag to release
+  #       required: true
+  #       type: string
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}

.github/workflows/release.yml

@@ -604,13 +604,13 @@ jobs:
         with:
           name: test-sqlsmith
 
-  bindings_python:
-    if: inputs.stable
-    needs: create_release
-    uses: ./.github/workflows/bindings.python.yml
-    secrets: inherit
-    with:
-      tag: ${{ needs.create_release.outputs.version }}
+  # bindings_python:
+  #   if: inputs.stable
+  #   needs: create_release
+  #   uses: ./.github/workflows/bindings.python.yml
+  #   secrets: inherit
+  #   with:
+  #     tag: ${{ needs.create_release.outputs.version }}
 
   notify:
     runs-on: ubuntu-latest

Cargo.toml

@@ -105,7 +105,7 @@ members = [
     "src/meta/service",
     # sqllogictest
     "tests/sqllogictests",
-    "src/bendpy",
+    # "src/bendpy",
     # sqlsmith
     "src/tests/sqlsmith",
 ]

src/bendpy/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "databend"
-requires-python = ">=3.7"
+requires-python = ">=3.12"
 description = "Databend Python Binding"
 classifiers = [
     "Programming Language :: Rust",

src/binaries/Cargo.toml

@@ -14,6 +14,7 @@ memory-profiling = [
     "databend-query/memory-profiling",
     "databend-common-base/memory-profiling",
 ]
+python-udf = ["databend-query/python-udf"]
 simd = ["databend-meta/simd", "databend-query/simd"]
 z3-prove = ["databend-query/z3-prove"]
 jemalloc = ["databend-common-base/jemalloc"]

src/common/exception/src/exception_code.rs

@@ -191,6 +191,7 @@ build_exceptions! {
     ///
     /// For example: license key is expired
     LicenseKeyInvalid(1402),
+    EnterpriseFeatureNotEnable(1403),
 
     BackgroundJobAlreadyExists(1501),
     UnknownBackgroundJob(1502),

src/query/service/Cargo.toml

@@ -14,7 +14,7 @@ test = true
 [features]
 default = ["simd", "z3-prove"]
 simd = ["databend-common-arrow/simd"]
-
+python-udf = ["arrow-udf-python"]
 z3-prove = ["databend-common-sql/z3-prove"]
 disable_initial_exec_tls = ["databend-common-base/disable_initial_exec_tls"]
 
@@ -103,6 +103,7 @@ jsonb = { workspace = true }
 
 # Crates.io dependencies
 arrow-udf-js = { workspace = true }
+arrow-udf-python = { package = "arrow-udf-python", git = "https://github.com/datafuse-extras/arrow-udf", rev = "d0a21f0", optional = true }
 arrow-udf-wasm = { workspace = true }
 
 arrow-array = { workspace = true }