feat(spdk): updating to spdk 24.01

Signed-off-by: Dmitry Savitskiy <[email protected]>

Cargo.toml

@@ -7,7 +7,7 @@ build = "build.rs"
 license = "Apache2"
 
 [build-dependencies]
-bindgen = "0.68.1"
+bindgen = "0.69.4"
 cc = "1.0.83"
 pkg-config = "0.3.27"
 

build.rs

@@ -52,6 +52,16 @@ fn get_target_dir() -> PathBuf {
     p
 }
 
+/// Returns nightly rustfmt path, if it can be found. Returns None oetherwise.
+fn rust_fmt_nightly() -> Option<PathBuf> {
+    let mut cmd = std::process::Command::new("rustup");
+    cmd.args(["which", "rustfmt", "--toolchain=nightly"]);
+
+    build_helpers::run_command(&mut cmd, "rustup", None)
+        .ok()
+        .and_then(|r| r.1.first().map(PathBuf::from))
+}
+
 /// Returns absolute path for SPDK library.
 fn get_spdk_path() -> Result<PathBuf, Error> {
     let spdk_path = match env::var_os("SPDK_PATH") {
@@ -159,6 +169,7 @@ fn configure_spdk() -> Result<LibraryConfig, Error> {
         "spdk_event_sock",
         "spdk_event_vmd",
         "spdk_nvmf",
+        "spdk_util",
     ])?;
 
     spdk_lib.find_lib("spdk_syslibs")?;
@@ -253,7 +264,7 @@ fn main() {
     println!("\nCompiling SPDK helpers...");
     match compile_spdk_helpers(&inc_dirs) {
         Ok(_) => {
-            println!("Successfully compiled SPDK helpers");
+            println!("Successfully compiled SPDK helpers\n");
         }
         Err(e) => {
             eprintln!("\nFailed to complie SPDK helpers: {e}\n");
@@ -332,6 +343,13 @@ fn main() {
             macros,
         }));
 
+    // Use nightly rustfmt if it is possible.
+    let bindings = if let Some(rust_fmt) = rust_fmt_nightly() {
+        bindings.with_rustfmt(rust_fmt)
+    } else {
+        bindings
+    };
+
     #[cfg(target_arch = "x86_64")]
     let bindings = bindings.clang_arg("-march=nehalem");
 

build_helpers/library_config.rs

@@ -69,6 +69,7 @@ pub fn append_path_var<V: AsRef<OsStr>>(var_name: V, new_path: &Path) {
 }
 
 /// Library dependency.
+#[derive(Debug, Clone)]
 pub struct Library {
     /// Short library name (e.g. `z` for `-lz`).
     lib_name: OsString,
@@ -319,27 +320,63 @@ impl LibraryConfig {
     ) -> Result<(), Error> {
         let lib_name = OsString::from(&lib_name);
         let library = self.cfg.probe(lib_name.to_str().unwrap())?;
+
+        println!("Found pkg config dependencies for {lib_name:?}:");
+
         for name in &library.libs {
             let lib = Library::new(name, &lib_name, self);
+            println!("    link lib {n:?}", n = lib.pkg_name);
+            if self.is_lib_needed(&lib) {
+                self.add_lib(lib);
+            }
+        }
+
+        for file_path in &library.link_files {
+            println!("    link file {file_path:?}");
+
+            let mut dep_lib_name = OsString::from(":");
+            dep_lib_name.push(file_path.file_name().unwrap());
+
+            let lib = Library::new(&dep_lib_name, &lib_name, self);
 
-            if !self.excluded.contains(&lib.lib_name)
-                && !self.libs.contains(&lib)
-            {
-                self.libs.insert(lib);
+            if self.is_lib_needed(&lib) {
+                self.add_lib_path(file_path.parent().unwrap());
+                self.add_lib(lib);
             }
         }
 
         for path in &library.link_paths {
-            self.lib_paths.insert(path.clone());
+            println!("    lib path {path:?}");
+            self.add_lib_path(path);
         }
 
         for path in &library.include_paths {
+            println!("    inc path {path:?}");
             self.inc_paths.insert(path.clone());
         }
 
         Ok(())
     }
 
+    /// Checks if the lib is needed.
+    pub fn is_lib_needed(&self, lib: &Library) -> bool {
+        !self.excluded.contains(&lib.lib_name) && !self.libs.contains(lib)
+    }
+
+    /// Adds a libs.
+    pub fn add_lib(&mut self, lib: Library) {
+        self.libs.insert(lib);
+    }
+
+    /// Adds a lib search path.
+    pub fn add_lib_path<T: AsRef<Path>>(&mut self, path: T) {
+        if !self.lib_paths.contains(path.as_ref()) {
+            let path = path.as_ref().to_owned();
+            println!("Added lib search path: {path:?}");
+            self.lib_paths.insert(path);
+        }
+    }
+
     /// Finds all given libraries via pkg-config.
     pub fn find_libs(&mut self, libs: &[&str]) -> Result<(), Error> {
         for name in libs.iter() {
@@ -370,17 +407,17 @@ impl LibraryConfig {
     /// TODO
     #[allow(dead_code)]
     pub fn dump(&self) {
-        println!("**** Found libraries:");
+        println!("**** Will link against libraries (A: archive, S: system):");
         for lib in self.libs.iter() {
             println!("    {lib}");
         }
 
-        println!("**** Found library paths:");
+        println!("**** Will use library paths:");
         for p in self.lib_paths.iter() {
             println!("    {}", p.to_str().unwrap());
         }
 
-        println!("**** Found include paths:");
+        println!("**** Will use include paths:");
         for p in self.inc_paths.iter() {
             println!("    {}", p.to_str().unwrap());
         }

helpers/logwrapper.h

@@ -1,5 +1,7 @@
 #include <stddef.h>
 #include <stdarg.h>
+
+#include "spdk_common.h"
 #include <spdk/log.h>
 
 typedef void spdk_rs_logger(int level, const char *file, const int line,

helpers/nvme_helper.h

@@ -1,6 +1,7 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "spdk_common.h"
 #include <spdk/bdev.h>
 #include <spdk/nvme.h>
 
@@ -18,5 +19,5 @@ uint16_t *nvme_status_raw_get(struct spdk_nvme_cpl *cpl);
 
 int
 spdk_bdev_nvme_admin_passthru_ro(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
-			      const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,
-			      spdk_bdev_io_completion_cb cb, void *cb_arg);
+				 const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,
+				 spdk_bdev_io_completion_cb cb, void *cb_arg);

helpers/spdk_common.h

@@ -0,0 +1,10 @@
+#ifndef MAYASTOR_SPDK_COMMON_H
+#define MAYASTOR_SPDK_COMMON_H
+
+#include <spdk/assert.h>
+
+#ifndef assert
+#define assert(expression) ((void)0)
+#endif
+
+#endif

helpers/spdk_helper.c

@@ -1,12 +1,7 @@
 
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wimplicit-function-declaration"
-
 #include "spdk_helper.h"
 #include <spdk/thread.h>
 
-#pragma GCC diagnostic pop
-
 void *spdk_rs_io_channel_get_ctx(struct spdk_io_channel *ch)
 {
 	return spdk_io_channel_get_ctx(ch);

helpers/spdk_helper.h

@@ -2,6 +2,7 @@
 #ifndef MAYASTOR_SPDK_HELPER_H
 #define MAYASTOR_SPDK_HELPER_H
 
+#include "spdk_common.h"
 #include <spdk/thread.h>
 
 /**

src/bdev_builder.rs

@@ -197,6 +197,7 @@ where
                 max_unmap_segments: Default::default(),
                 max_write_zeroes: Default::default(),
                 max_copy: Default::default(),
+                max_rw_size: Default::default(),
                 uuid: self.uuid.unwrap_or_else(Uuid::generate).into_raw(),
                 md_len: Default::default(),
                 md_interleave: Default::default(),

src/bdevs/bdev_nvme.rs

@@ -0,0 +1,56 @@
+use futures::channel::oneshot::{self, Canceled};
+use nix::errno::Errno;
+
+use crate::{
+    ffihelper::{
+        cb_arg,
+        copy_str_with_null,
+        done_errno_cb,
+        drop_cb_arg,
+        ErrnoResult,
+        IntoCString,
+    },
+    libspdk::{bdev_nvme_delete, nvme_path_id, SPDK_NVME_TRANSPORT_PCIE},
+};
+
+/// Async wrapper for `bdev_nvme_delete`.
+/// `bdev_nvme_delete` differs from other bdev_*_delete function family,
+/// as it may return errno instead calling the callback (which is optional).
+///
+/// # Arguments
+///
+/// * `name`: Controller name.
+///
+/// * `path_id`: Controller path ID. If not given, `name` is used for traddr,
+///   and SPDK_NVME_TRANSPORT_PCIE for trtype.
+pub async fn bdev_nvme_delete_async(
+    name: &str,
+    path_id: Option<nvme_path_id>,
+) -> Result<ErrnoResult<()>, Canceled> {
+    let path_id = path_id.unwrap_or_else(|| {
+        let mut path_id = nvme_path_id::default();
+        copy_str_with_null(name, &mut path_id.trid.traddr);
+        path_id.trid.trtype = SPDK_NVME_TRANSPORT_PCIE;
+        path_id
+    });
+
+    let (s, r) = oneshot::channel::<ErrnoResult<()>>();
+    let arg = cb_arg(s);
+
+    let errno = unsafe {
+        bdev_nvme_delete(
+            name.to_string().into_cstring().as_ptr(),
+            &path_id,
+            Some(done_errno_cb),
+            arg,
+        )
+    };
+
+    // `bdev_nvme_delete` failed to run: callback won't be called.
+    if errno < 0 {
+        drop_cb_arg::<()>(arg);
+        return Ok(Err(Errno::from_i32(-errno)));
+    }
+
+    r.await
+}

src/bdevs/mod.rs

@@ -0,0 +1,5 @@
+///! Wrappers for individual Bdev types.
+
+pub mod bdev_nvme;
+
+pub use bdev_nvme::bdev_nvme_delete_async;

src/ffihelper.rs

@@ -114,12 +114,23 @@ pub fn copy_cstr_to_buf_with_null(
 /// Result having Errno error.
 pub type ErrnoResult<T, E = Errno> = Result<T, E>;
 
-/// Construct callback argument for spdk async function.
+/// Constructs a callback argument for spdk async function.
 /// The argument is a oneshot sender channel for result of the operation.
+/// The pointer returned by this function is a raw pointer to
+/// a heap-allocated object, and it must be consumed by `done_cb`,
+/// `done_errno_cb`, or dropped explicitly by `drop_cb_arg`.
 pub fn cb_arg<T>(sender: oneshot::Sender<T>) -> *mut c_void {
     Box::into_raw(Box::new(sender)) as *const _ as *mut c_void
 }
 
+/// Drops a callback argument contructed by `cb_arg`.
+/// This is needed when the callback is known to be not called.
+pub fn drop_cb_arg<T>(sender_ptr: *mut c_void) {
+    let sender =
+        unsafe { Box::from_raw(sender_ptr as *mut oneshot::Sender<T>) };
+    drop(sender);
+}
+
 /// A generic callback for spdk async functions expecting to be called with
 /// single argument which is a sender channel to notify the other end about
 /// the result.

src/lib.rs

@@ -20,6 +20,7 @@ extern crate tracing;
 extern crate serde;
 extern crate serde_json;
 
+pub mod bdevs;
 pub mod cpu_cores;
 pub mod ffihelper;
 pub mod libspdk;

src/uuid.rs

@@ -32,7 +32,7 @@ impl Uuid {
     }
 
     /// Consumes the `Uuid` and returns its SPDK internal representation.
-    pub(crate) fn into_raw(self) -> spdk_uuid {
+    pub fn into_raw(self) -> spdk_uuid {
         self.inner
     }
 

wrapper.h

@@ -1,3 +1,10 @@
+
+#include <spdk/assert.h>
+
+#ifndef assert
+#define assert(expression) ((void)0)
+#endif
+
 #include <bdev/aio/bdev_aio.h>
 #include <bdev/crypto/vbdev_crypto.h>
 #include <bdev/error/vbdev_error.h>
@@ -30,6 +37,7 @@
 #include <spdk/rpc.h>
 #include <spdk/scsi.h>
 #include <spdk/thread.h>
+#include <spdk/trace.h>
 #include <spdk/uuid.h>
 #include <spdk/version.h>
 #include <spdk_internal/event.h>