Bump dora v0.3.7 (#702)

This is a release PR of 0.3.7
dora-rs · Nov 18, 2024 · efe6cc9 · efe6cc9
2 parents a9f6433 + 76e9c60
commit efe6cc9
Show file tree

Hide file tree

Showing 45 changed files with 436 additions and 290 deletions.
diff --git a/.github/workflows/node-hub-ci-cd.yml b/.github/workflows/node-hub-ci-cd.yml
@@ -1,6 +1,7 @@
 name: node-hub
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
@@ -130,18 +131,31 @@ jobs:
         run: |
           dir=$(pwd)
           base_dir=$(basename "$dir")
+          ignored_folders=("dora-parler")
 
-          if [[ -f "Cargo.toml" && -f "pyproject.toml" ]]; then
-            echo "Publishing $dir using maturin..."
-            poetry publish
+          if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
+              echo "Skipping $base_dir as there is a hf model fetching issue..."
           else
-            if [ -f "pyproject.toml" ]; then
-              echo "Publishing $dir using Poetry..."
-              poetry publish --build
+            if [[ -f "Cargo.toml" && -f "pyproject.toml" ]]; then
+              echo "Publishing $dir using maturin..."
+              pip3 install "maturin[patchelf]"                                                                             
+              maturin publish  --skip-existing
+            else
+              if [ -f "pyproject.toml" ]; then
+                echo "Publishing $dir using Poetry..."
+                poetry publish --build  --skip-existing
+              fi
+
+              if [ -f "Cargo.toml" ]; then
+                echo "Publishing $dir using Cargo..."
+                package_name=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].name')
+                version=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[0].version')
+                if cargo search "$package_name" | grep -q "^$package_name = \"$version\""; then
+                    echo "Package '$package_name' version '$version' already exists on crates.io. Skipping publish."
+                else
+                    echo "Publishing package '$package_name' version '$version'..."
+                    cargo publish
+                fi
+              fi
             fi
           fi
-
-          if [ -f "Cargo.toml" ]; then
-            echo "Publishing $dir using Cargo..."
-            cargo publish
-          fi
diff --git a/.github/workflows/pip-release.yml b/.github/workflows/pip-release.yml
@@ -41,7 +41,9 @@ jobs:
           # target: ppc64le
         repository:
           - path: apis/python/node
+            name: dora-node-api
           - path: binaries/cli
+            name: dora-rs-cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -63,16 +65,8 @@ jobs:
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-linux-${{ matrix.platform.target }}
+          name: ${{ matrix.repository.name }}-linux-${{ matrix.platform.target }}
           path: ${{ matrix.repository.path }}/dist
-      - name: Upload to release
-        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ matrix.repository.path }}/dist/*
-          tag: ${{ github.ref }}
-          file_glob: true
 
   musllinux:
     runs-on: ${{ matrix.platform.runner }}
@@ -87,7 +81,9 @@ jobs:
             target: aarch64
         repository:
           - path: apis/python/node
+            name: dora-node-api
           - path: binaries/cli
+            name: dora-rs-cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -105,16 +101,8 @@ jobs:
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-musllinux-${{ matrix.platform.target }}
+          name: ${{ matrix.repository.name }}-musllinux-${{ matrix.platform.target }}
           path: ${{ matrix.repository.path }}/dist
-      - name: Upload to release
-        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ matrix.repository.path }}/dist/*
-          tag: ${{ github.ref }}
-          file_glob: true
 
   musleabi:
     runs-on: ubuntu-22.04
@@ -129,7 +117,9 @@ jobs:
           ]
         repository:
           - path: apis/python/node
+            name: dora-node-api
           - path: binaries/cli
+            name: dora-rs-cli
     container:
       image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }}
       env:
@@ -151,16 +141,8 @@ jobs:
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v3
         with:
-          name: wheels-musllinux-${{ matrix.platform.target }}
+          name: ${{ matrix.repository.name }}-musllinux-${{ matrix.platform.target }}
           path: ${{ matrix.repository.path }}/dist
-      - name: Upload to release
-        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ matrix.repository.path }}/dist/*
-          tag: ${{ github.ref }}
-          file_glob: true
 
   windows:
     runs-on: ${{ matrix.platform.runner }}
@@ -171,7 +153,9 @@ jobs:
             target: x64
         repository:
           - path: apis/python/node
+            name: dora-node-api
           - path: binaries/cli
+            name: dora-rs-cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -189,16 +173,8 @@ jobs:
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-windows-${{ matrix.platform.target }}
+          name: ${{ matrix.repository.name }}-windows-${{ matrix.platform.target }}
           path: ${{ matrix.repository.path }}/dist
-      - name: Upload to release
-        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ matrix.repository.path }}/dist/*
-          tag: ${{ github.ref }}
-          file_glob: true
 
   macos:
     runs-on: ${{ matrix.platform.runner }}
@@ -211,7 +187,9 @@ jobs:
             target: aarch64
         repository:
           - path: apis/python/node
+            name: dora-node-api
           - path: binaries/cli
+            name: dora-rs-cli
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -228,24 +206,18 @@ jobs:
         if: github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
-          name: wheels-macos-${{ matrix.platform.target }}
+          name: ${{ matrix.repository.name }}-macos-${{ matrix.platform.target }}
           path: ${{ matrix.repository.path }}/dist
-      - name: Upload to release
-        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: ${{ matrix.repository.path }}/dist/*
-          tag: ${{ github.ref }}
-          file_glob: true
 
   sdist:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
         repository:
           - path: apis/python/node
+            name: dora-node-api
           - path: binaries/cli
+            name: dora-rs-cli
     steps:
       - uses: actions/checkout@v3
       - name: Build sdist
@@ -266,6 +238,13 @@ jobs:
     runs-on: ubuntu-22.04
     if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')
     needs: [linux, musllinux, musleabi, windows, macos, sdist]
+    strategy:
+      matrix:
+        repository:
+          - path: apis/python/node
+            name: dora-node-api
+          - path: binaries/cli
+            name: dora-rs-cli
     steps:
       - uses: actions/download-artifact@v4
       - name: Publish to PyPI
@@ -274,4 +253,4 @@ jobs:
           MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASS }}
         with:
           command: upload
-          args: --non-interactive --skip-existing wheels-*/*
+          args: --non-interactive --skip-existing ${{ matrix.repository.name }}-*/*
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -42,6 +42,8 @@ jobs:
 
       - name: "Publish packages on `crates.io`"
         if: runner.os == 'Linux'
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
         run: |
 
           # Publishing those crates from outer crates with no dependency to inner crates
@@ -52,33 +54,50 @@ jobs:
           # We should preferably test pre-releases before testing releases as
           # cargo publish might catch release issues that the workspace manages to fix using 
           # workspace crates.
+          publish_if_not_exists() {
+              local package_name=$1
+              local version=$(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | select(.name=="'"$package_name"'") | .version')
+
+              if [[ -z $version ]]; then
+                  echo "error: package '$package_name' not found in the workspace."
+                  return 1
+              fi
+
+              if cargo search "$package_name" | grep -q "^$package_name = \"$version\""; then
+                  echo "package '$package_name' version '$version' already exists on crates.io. skipping publish."
+              else
+                  echo "publishing package '$package_name' version '$version'..."
+                  cargo publish --package "$package_name"
+              fi
+          }
 
-          # Publish libraries crates
-          cargo publish -p dora-tracing --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-metrics --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-download --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-core --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
           # the dora-message package is versioned separately, so this publish command might fail if the version is already published
-          cargo publish -p dora-message --token ${{ secrets.CARGO_REGISTRY_TOKEN }} || true
-          cargo publish -p communication-layer-pub-sub --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p communication-layer-request-reply --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p shared-memory-server --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-arrow-convert --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          publish_if_not_exists dora-message
+
+          # Publish libraries crates
+          publish_if_not_exists dora-tracing
+          publish_if_not_exists dora-metrics 
+          publish_if_not_exists dora-download 
+          publish_if_not_exists dora-core
+          publish_if_not_exists communication-layer-pub-sub
+          publish_if_not_exists communication-layer-request-reply
+          publish_if_not_exists shared-memory-server
+          publish_if_not_exists dora-arrow-convert
 
           # Publish rust API
-          cargo publish -p dora-operator-api-macros --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-operator-api-types --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-operator-api --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-node-api --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-operator-api-python --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-operator-api-c --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-node-api-c --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          publish_if_not_exists dora-operator-api-macros
+          publish_if_not_exists dora-operator-api-types
+          publish_if_not_exists dora-operator-api
+          publish_if_not_exists dora-node-api
+          publish_if_not_exists dora-operator-api-python
+          publish_if_not_exists dora-operator-api-c
+          publish_if_not_exists dora-node-api-c
 
           # Publish binaries crates
-          cargo publish -p dora-coordinator --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-runtime --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-daemon --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
-          cargo publish -p dora-cli --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          publish_if_not_exists dora-coordinator
+          publish_if_not_exists dora-runtime
+          publish_if_not_exists dora-daemon
+          publish_if_not_exists dora-cli
 
   unix:
     runs-on: ${{ matrix.platform.runner }}

diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,11 @@ examples/**/*.txt
 **/*.arrow
 *.pt
 
+# Remove hdf and stl files
+*.stl
+*.STL
+*.hdf5
+
 # Removing images.
 *.jpg
 *.mp3