From 9349c97b70fa60a854e82bc09249d8826bff934a Mon Sep 17 00:00:00 2001 From: Ksenia Vazhdaeva Date: Fri, 23 Aug 2024 12:26:49 +0700 Subject: [PATCH] Issue #6. Update dependencies --- .github/workflows/docs.yml | 11 +++++----- .github/workflows/rocksq.yml | 20 ++++++++--------- docker/Dockerfile.manylinux_2_28_ARM64 | 2 +- docker/Dockerfile.manylinux_2_28_X64 | 2 +- queue_py/Cargo.toml | 6 +++--- queue_py/src/blocking.rs | 18 +++++++++------- queue_py/src/lib.rs | 30 ++++++++++++++++++-------- queue_py/src/nonblocking.rs | 18 +++++++++------- queue_rs/Cargo.toml | 2 +- 9 files changed, 62 insertions(+), 47 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ba94e99..a7b97e1 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,9 +27,9 @@ jobs: build-docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' @@ -54,7 +54,7 @@ jobs: --directory build/html -cvf artifact.tar . - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: github-pages path: docs/artifact.tar @@ -70,9 +70,8 @@ jobs: steps: - name: Setup Pages - uses: actions/configure-pages@v1 + uses: actions/configure-pages@v5 - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v1 - + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/rocksq.yml b/.github/workflows/rocksq.yml index c62393f..e1e341d 100644 --- a/.github/workflows/rocksq.yml +++ b/.github/workflows/rocksq.yml @@ -39,16 +39,16 @@ jobs: - self-hosted - ${{ matrix.runner }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up docker buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Build docker image - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: file: ${{ matrix.docker_file }} platforms: ${{ matrix.arch }} @@ -61,9 +61,9 @@ jobs: run: docker run --rm -v $(pwd)/distfiles:/tmp rocksq cp -R /opt/dist /tmp - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-${{ matrix.runner }} path: distfiles/dist - name: Publish to PyPI @@ -82,8 +82,8 @@ jobs: target: [x64] python: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} architecture: ${{ matrix.target }} @@ -95,9 +95,9 @@ jobs: args: --release --out dist --find-interpreter sccache: 'true' - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: wheels + name: wheels-${{ matrix.python }} path: queue_py/dist - name: Publish to PyPI if: "startsWith(github.ref, 'refs/tags/')" diff --git a/docker/Dockerfile.manylinux_2_28_ARM64 b/docker/Dockerfile.manylinux_2_28_ARM64 index af966f5..b33bb26 100644 --- a/docker/Dockerfile.manylinux_2_28_ARM64 +++ b/docker/Dockerfile.manylinux_2_28_ARM64 @@ -1,4 +1,4 @@ -FROM ghcr.io/insight-platform/manylinux_2_28_arm64:v0.0.3 as chef +FROM ghcr.io/insight-platform/manylinux_2_28_arm64:v0.0.4 as chef FROM chef as planner diff --git a/docker/Dockerfile.manylinux_2_28_X64 b/docker/Dockerfile.manylinux_2_28_X64 index 67f3170..3ca3d3c 100644 --- a/docker/Dockerfile.manylinux_2_28_X64 +++ b/docker/Dockerfile.manylinux_2_28_X64 @@ -1,4 +1,4 @@ -FROM ghcr.io/insight-platform/manylinux_2_28_x64:v0.0.3 as chef +FROM ghcr.io/insight-platform/manylinux_2_28_x64:v0.0.4 as chef FROM chef as planner diff --git a/queue_py/Cargo.toml b/queue_py/Cargo.toml index b511ff2..262b9f1 100644 --- a/queue_py/Cargo.toml +++ b/queue_py/Cargo.toml @@ -19,14 +19,14 @@ crate-type = ["cdylib", "lib"] queue_rs = { path = "../queue_rs" } [dependencies.pyo3] -version = "0.20" +version = "0.22.2" [dev-dependencies] serial_test = "3" [build-dependencies] -pyo3-build-config = "0.19" -cbindgen = "0.24" +pyo3-build-config = "0.22.2" +cbindgen = "0.27.0" [package.metadata.maturin] python-source = "python" diff --git a/queue_py/src/blocking.rs b/queue_py/src/blocking.rs index a839365..3bf8fd8 100644 --- a/queue_py/src/blocking.rs +++ b/queue_py/src/blocking.rs @@ -1,7 +1,7 @@ -use crate::StartPosition; +use crate::{pylist_to_vec_of_byte_vec, value_as_slice, StartPosition}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; -use pyo3::types::PyBytes; +use pyo3::types::{PyBytes, PyList}; use queue_rs::mpmc; use rocksdb::Options; use std::time::Duration; @@ -60,8 +60,9 @@ impl PersistentQueueWithCapacity { /// None /// #[pyo3(signature = (items, no_gil = true))] - fn push(&self, items: Vec<&PyBytes>, no_gil: bool) -> PyResult<()> { - let data = items.iter().map(|e| e.as_bytes()).collect::>(); + fn push(&self, items: &Bound<'_, PyList>, no_gil: bool) -> PyResult<()> { + let items = pylist_to_vec_of_byte_vec(items); + let data = value_as_slice(&items); Python::with_gil(|py| { let f = || { self.0 @@ -110,7 +111,7 @@ impl PersistentQueueWithCapacity { results .into_iter() .map(|r| { - PyBytes::new_with(py, r.len(), |b: &mut [u8]| { + PyBytes::new_bound_with(py, r.len(), |b: &mut [u8]| { b.copy_from_slice(&r); Ok(()) }) @@ -232,8 +233,9 @@ impl MpmcQueue { /// None /// #[pyo3(signature = (items, no_gil = true))] - fn add(&self, items: Vec<&PyBytes>, no_gil: bool) -> PyResult<()> { - let data = items.iter().map(|e| e.as_bytes()).collect::>(); + fn add(&self, items: &Bound<'_, PyList>, no_gil: bool) -> PyResult<()> { + let items = pylist_to_vec_of_byte_vec(items); + let data = value_as_slice(&items); Python::with_gil(|py| { let f = || { self.0 @@ -301,7 +303,7 @@ impl MpmcQueue { results .into_iter() .map(|r| { - PyBytes::new_with(py, r.len(), |b: &mut [u8]| { + PyBytes::new_bound_with(py, r.len(), |b: &mut [u8]| { b.copy_from_slice(&r); Ok(()) }) diff --git a/queue_py/src/lib.rs b/queue_py/src/lib.rs index 566de17..7088ecf 100644 --- a/queue_py/src/lib.rs +++ b/queue_py/src/lib.rs @@ -1,6 +1,6 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyList}; use pyo3::wrap_pymodule; mod blocking; @@ -54,22 +54,22 @@ fn remove_mpmc_queue(path: &str) -> PyResult<()> { .map_err(|e| PyRuntimeError::new_err(format!("Failed to remove mpmc queue: {}", e))) } -#[pyclass] +#[pyclass(eq, eq_int)] #[derive(PartialEq, Copy, Clone)] enum StartPosition { - Oldest, - Newest, + Oldest = 0, + Newest = 1, } #[pymodule] -fn rocksq_blocking(_: Python, m: &PyModule) -> PyResult<()> { +fn rocksq_blocking(_: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; Ok(()) } #[pymodule] -fn rocksq_nonblocking(_: Python, m: &PyModule) -> PyResult<()> { +fn rocksq_nonblocking(_: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -82,7 +82,7 @@ fn rocksq_nonblocking(_: Python, m: &PyModule) -> PyResult<()> { } #[pymodule] -fn rocksq(py: Python, m: &PyModule) -> PyResult<()> { +fn rocksq(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(version, m)?)?; m.add_function(wrap_pyfunction!(remove_queue, m)?)?; m.add_function(wrap_pyfunction!(remove_mpmc_queue, m)?)?; @@ -92,11 +92,23 @@ fn rocksq(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; - let sys = PyModule::import(py, "sys")?; - let sys_modules: &PyDict = sys.getattr("modules")?.downcast()?; + let sys = PyModule::import_bound(py, "sys")?; + let attr_modules = sys.getattr("modules")?; + let sys_modules: &Bound<'_, PyDict> = attr_modules.downcast::()?; sys_modules.set_item("rocksq.blocking", m.getattr("rocksq_blocking")?)?; sys_modules.set_item("rocksq.nonblocking", m.getattr("rocksq_nonblocking")?)?; Ok(()) } + +fn pylist_to_vec_of_byte_vec(items: &Bound<'_, PyList>) -> Vec> { + items + .iter() + .map(|e| e.extract::>().unwrap()) + .collect::>() +} + +fn value_as_slice(items: &[Vec]) -> Vec<&[T]> { + items.iter().map(|e| e.as_slice()).collect::>() +} diff --git a/queue_py/src/nonblocking.rs b/queue_py/src/nonblocking.rs index 712fe1d..f3c1bf6 100644 --- a/queue_py/src/nonblocking.rs +++ b/queue_py/src/nonblocking.rs @@ -1,7 +1,7 @@ -use crate::StartPosition; +use crate::{pylist_to_vec_of_byte_vec, value_as_slice, StartPosition}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; -use pyo3::types::PyBytes; +use pyo3::types::{PyBytes, PyList}; use queue_rs::mpmc; use rocksdb::Options; use std::time::Duration; @@ -38,7 +38,7 @@ impl ResponseVariant { results .iter() .map(|r| { - PyBytes::new_with(py, r.len(), |b: &mut [u8]| { + PyBytes::new_bound_with(py, r.len(), |b: &mut [u8]| { b.copy_from_slice(r); Ok(()) }) @@ -227,8 +227,9 @@ impl PersistentQueueWithCapacity { /// the response object is only useful to call for `is_ready()`. /// #[pyo3(signature = (items, no_gil = true))] - fn push(&self, items: Vec<&PyBytes>, no_gil: bool) -> PyResult { - let data = items.iter().map(|e| e.as_bytes()).collect::>(); + fn push(&self, items: &Bound<'_, PyList>, no_gil: bool) -> PyResult { + let items = pylist_to_vec_of_byte_vec(items); + let data = value_as_slice(&items); Python::with_gil(|py| { let f = || { self.0 @@ -380,7 +381,7 @@ impl MpmcResponseVariant { results .iter() .map(|r| { - PyBytes::new_with(py, r.len(), |b: &mut [u8]| { + PyBytes::new_bound_with(py, r.len(), |b: &mut [u8]| { b.copy_from_slice(r); Ok(()) }) @@ -617,8 +618,9 @@ impl MpmcQueue { /// the response object is only useful to call for `is_ready()`. /// #[pyo3(signature = (items, no_gil = true))] - fn add(&self, items: Vec<&PyBytes>, no_gil: bool) -> PyResult { - let data = items.iter().map(|e| e.as_bytes()).collect::>(); + fn add(&self, items: &Bound<'_, PyList>, no_gil: bool) -> PyResult { + let items = pylist_to_vec_of_byte_vec(items); + let data = value_as_slice(&items); Python::with_gil(|py| { let f = || { self.0 diff --git a/queue_rs/Cargo.toml b/queue_rs/Cargo.toml index 09f090f..e99bc93 100644 --- a/queue_rs/Cargo.toml +++ b/queue_rs/Cargo.toml @@ -20,7 +20,7 @@ crate-type = ["cdylib", "lib"] [dependencies] anyhow = "1" parking_lot = "0.12" -crossbeam-channel = "0.5.8" +crossbeam-channel = "0.5.13" bincode = "2.0.0-rc.3" chrono = "0.4.38"