diff --git a/shawk/.github/workflows/release.yml b/.github/workflows/release.yml similarity index 100% rename from shawk/.github/workflows/release.yml rename to .github/workflows/release.yml diff --git a/shawk/Cargo.lock b/Cargo.lock similarity index 92% rename from shawk/Cargo.lock rename to Cargo.lock index d66458e71..e9976e221 100644 --- a/shawk/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,21 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.13" @@ -100,9 +115,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bumpalo" @@ -128,6 +143,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets 0.52.4", +] + [[package]] name = "clap" version = "4.5.3" @@ -320,6 +349,16 @@ dependencies = [ "slab", ] +[[package]] +name = "gethostname" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0176e0459c2e4a1fe232f984bca6890e681076abb9934f6cea7c326f3fc47818" +dependencies = [ + "libc", + "windows-targets 0.48.5", +] + [[package]] name = "getrandom" version = "0.2.12" @@ -439,6 +478,29 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.5.0" @@ -480,6 +542,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-dir-list" +version = "0.1.0" +dependencies = [ + "chrono", + "gethostname", + "serde", + "serde_json", + "users", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -498,7 +571,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", "redox_syscall", ] @@ -565,6 +638,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + [[package]] name = "object" version = "0.32.2" @@ -586,7 +668,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -694,9 +776,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.26" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64", "bytes", @@ -761,11 +843,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -1105,6 +1187,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "users" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" +dependencies = [ + "libc", + "log", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -1208,6 +1300,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.4", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 000000000..7ebd49c4e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,25 @@ +[workspace] +resolver = '2' +members = [ + 'shawk', + 'json-dir-list', +] + +# generated by 'cargo dist init' +[profile.dist] +inherits = "release" +debug = true +split-debuginfo = "packed" + +# Config for 'cargo dist' +[workspace.metadata.dist] +# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.11.1" +# CI backends to support +ci = ["github"] +# The installers to generate for each app +installers = ["shell", "msi"] +# Target platforms to build apps for (Rust target-triple syntax) +targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] +# Publish jobs to run in CI +pr-run-mode = "plan" diff --git a/docs/plugin-sftp.md b/docs/plugin-sftp.md index b1093d6b5..b9b518371 100644 --- a/docs/plugin-sftp.md +++ b/docs/plugin-sftp.md @@ -153,13 +153,10 @@ structure using a `find` command to produce JSON output. This works about as well as one could hope. If file names, user names, or group names have characters that are now allowed in JSON strings, it does not go well. This tool provides a more robust alternative that does the JSON encoding correctly. To -install it: +install it, first, [install Rust](https://www.rust-lang.org/tools/install) and +then invoke: - sudo apt-get install build-essential pkg-config libjsoncpp-dev - autoreconf -i - ./configure --prefix=/install/dir - make - make install + cargo install --path . Once installed, in the `.sftp` configuration for Shesmu, the `"listCommand"` can be set to `"/install/dir/bin/json-dir-list"` to use this command instead. diff --git a/json-dir-list/Cargo.toml b/json-dir-list/Cargo.toml new file mode 100644 index 000000000..dd3e5de16 --- /dev/null +++ b/json-dir-list/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "json-dir-list" +version = "0.1.0" +edition = "2021" +repository = "https://github.com/oicr-gsi/shesmu" +authors = ["Andre Masella "] +description = "Scans directories for Shesmu" + +[package.metadata.wix] +upgrade-guid = "6C1F4D6E-D287-497C-A284-422E8C118523" +path-guid = "C77C620C-E21B-4DD6-BAD1-6D753D381C58" +license = false +eula = false + +[dependencies] +chrono = "^0.4" +gethostname = "^0.4" +serde = { version = "^1.0", features = ["derive"] } +serde_json = "^1.0" +users = "^0.11" diff --git a/json-dir-list/Makefile.am b/json-dir-list/Makefile.am deleted file mode 100644 index 04d03e723..000000000 --- a/json-dir-list/Makefile.am +++ /dev/null @@ -1,33 +0,0 @@ -# Define an empty variable for list termination -NULL = -ACLOCAL_AMFLAGS = -I m4 - -# Build the following binaries and put the in $prefix/bin -bin_PROGRAMS = \ - json-dir-list \ - $(NULL) -# Install the following manpages in $prefix/share/man/man1 -man1_MANS = \ - json-dir-list.1 \ - $(NULL) - -# These are all the source .cpp files for. Note the change of `-` to `_`. -json_dir_list_SOURCES = \ - json-dir-list.cpp \ - $(NULL) -# These are the compiler arguments for building -# We want (in order): -# - C++11 -# - debugging symbols to be included -# - optimisation level 2 -# - whatever junk jsoncpp wants -json_dir_list_CPPFLAGS = \ - -std=c++11 -g -O2 \ - $(JSON_CFLAGS) \ - $(NULL) -# These are the linker flags for building -# We need: -# - whatever junk jsoncpp needs -json_dir_list_LDADD = \ - $(JSON_LIBS) \ - $(NULL) diff --git a/json-dir-list/README.md b/json-dir-list/README.md index cbd804bb1..00e8a63e0 100644 --- a/json-dir-list/README.md +++ b/json-dir-list/README.md @@ -6,11 +6,7 @@ names have characters that are now allowed in JSON strings, it does not go well. This tool provides a more robust alternative that does the JSON encoding correctly. To install it: - sudo apt-get install build-essential pkg-config libjsoncpp-dev - autoreconf -i - ./configure --prefix=/install/dir - make - make install + cargo install --path . Once installed, in the `.sftp` configuration for Shesmu, the `"listCommand"` can be set to `"/install/dir/bin/json-dir-list"` to use this command instead. diff --git a/json-dir-list/configure.ac b/json-dir-list/configure.ac deleted file mode 100644 index 9158653a2..000000000 --- a/json-dir-list/configure.ac +++ /dev/null @@ -1,18 +0,0 @@ -AC_INIT([json-dir-list], [1.0], [andre.masella@oicr.on.ca]) -# Configure automake in a standard way -AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects]) -AC_CONFIG_MACRO_DIR([m4]) -AC_CONFIG_HEADERS(config.h) -# Find a functioning C++ compiler -AC_PROG_CXX_C_O -# Find a functioning libtool, the C/C++ linking wrapper that handles complex -# dependencies. This is typical for AutoTools. -AC_PROG_LIBTOOL - -# Detect jsoncpp and zip using pkg-config. The detected library information will be -# bound to variables starting with `JSON_` -PKG_CHECK_MODULES(JSON, [ jsoncpp ]) - -# Using the detected information, transform `Makefile.in` to `Makefile` -AC_CONFIG_FILES([Makefile]) -AC_OUTPUT diff --git a/json-dir-list/json-dir-list.1 b/json-dir-list/json-dir-list.1 deleted file mode 100644 index 7c2233d5f..000000000 --- a/json-dir-list/json-dir-list.1 +++ /dev/null @@ -1,17 +0,0 @@ -.TH json-dir-list 1 "Jul 2020" "1.0" "USER COMMANDS" -.SH NAME -json-dir-list \- Produce Shesmu-compatible JSON directory listings -.SH SYNOPSIS -.B json-dir-list directory ... -.I directory -.SH DESCRIPTION -Perform a recursive search for files in the directories specified in a format -compatible with Shesmu's \fBunix_file\fR input format in JSON. This is meant to -be available to Shesmu's SSH plugin so that it can pull in the file information -in a safe format. Normally, Shesmu tries to use a \fBfind\fR command to -generate a JSON description of the directory structure, but this requires all -file, user, and group names be valid JSON strings without escaping. This -command does correct JSON encoding no matter the names involved. -.SH SEE ALSO -.BR find (1). - diff --git a/json-dir-list/json-dir-list.cpp b/json-dir-list/json-dir-list.cpp deleted file mode 100644 index eb24de246..000000000 --- a/json-dir-list/json-dir-list.cpp +++ /dev/null @@ -1,110 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Convert a timespec to an approximate floating point value of second since -// the epoch; this is a terrible format, but it's what `find` gives us, so -// we're going with it. -static double to_seconds(timespec ts) { - return (double)ts.tv_sec + (ts.tv_nsec / 1E9); -} - -int main(int argc, const char **argv) { - // Figure out the hostname to report in the records we return - std::string host("unknown"); - char hostname[1024]; - if (gethostname(hostname, sizeof hostname) == 0) { - host = hostname; - } - - // Populate our queue with the directories provided by the user - std::deque roots; - for (auto i = 1; i < argc; i++) { - roots.push_back(argv[i]); - } - - const Json::LargestInt fetched = - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch()) - .count(); - - // The JSON library can't do streaming writes, so we're going to write the - // array manually, and write the objects using the library. - auto first = true; - std::cout << "["; - while (!roots.empty()) { - DIR *dir = nullptr; - errno = 0; - dir = opendir(roots.front().c_str()); - if (dir == nullptr) { - // We're just going to keep going if we encounter errors reading a - // directory - perror("opendir"); - roots.pop_front(); - continue; - } else { - struct dirent *entry = nullptr; - while ((entry = readdir(dir))) { - if (strcmp(entry->d_name, ".") == 0 || - strcmp(entry->d_name, "..") == 0) { - continue; - } - std::stringstream path; - path << roots.front() << "/" << entry->d_name; - auto is_dir = entry->d_type == DT_DIR; - struct stat info = {0}; - // We might get something we can't identify, in which case we need to do - // stat to determine if it's a directory. If it isn't a directory, we - // need to stat it anyway for later. - if (entry->d_type == DT_UNKNOWN || !is_dir) { - if (stat(path.str().c_str(), &info) != 0) { - continue; - } - is_dir = S_ISDIR(info.st_mode); - } - - if (is_dir) { - // Any child directories should be explored later - roots.push_back(path.str()); - } else { - // For anything that's a file (or pipe or symlink or whatever), write - // out a record - Json::Value record(Json::objectValue); - struct passwd *pw = getpwuid(info.st_uid); - struct group *gr = getgrgid(info.st_gid); - record["fetched"] = fetched; - record["file"] = path.str(); - record["size"] = (Json::LargestInt)info.st_size; - record["atime"] = to_seconds(info.st_atim); - record["ctime"] = to_seconds(info.st_ctim); - record["mtime"] = to_seconds(info.st_mtim); - record["user"] = pw->pw_name; - record["group"] = gr->gr_name; - record["perms"] = (int)info.st_mode; - record["host"] = host; - - if (first) { - first = false; - } else { - std::cout << ","; - } - std::cout << record; - } - } - } - closedir(dir); - roots.pop_front(); - } - std::cout << "]"; - - return 0; -} diff --git a/json-dir-list/src/main.rs b/json-dir-list/src/main.rs new file mode 100644 index 000000000..50f699aa2 --- /dev/null +++ b/json-dir-list/src/main.rs @@ -0,0 +1,102 @@ +use serde::ser::SerializeSeq; +use serde::{Serialize, Serializer}; +use std::collections::VecDeque; +use std::os::linux::fs::MetadataExt; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::time::SystemTime; + +#[derive(Serialize)] +struct UnixFileData<'a> { + atime: f64, + + ctime: f64, + + fetched: &'a str, + + file: &'a str, + + group: &'a str, + + host: &'a str, + + mtime: f64, + + perms: u32, + + size: u64, + + user: &'a str, +} +// Convert a timespec to an approximate floating point value of second since +// the epoch; this is a terrible format, but it's what `find` gives us, so +// we're going with it. +fn to_seconds(time: std::io::Result) -> f64 { + match time { + Err(_) => 0.0, + Ok(time) => { + let (duration, multiplier) = match time.duration_since(SystemTime::UNIX_EPOCH) { + Err(err) => (err.duration(), -1.0), + Ok(duration) => (duration, 1.0), + }; + duration.as_secs_f64() * multiplier + } + } +} + +pub fn main() { + let hostname = gethostname::gethostname() + .into_string() + .expect("Host name is not valid Unicode"); + + let mut roots: VecDeque = std::env::args().map(|dir| PathBuf::from(dir)).collect(); + + let fetched = chrono::Utc::now().to_rfc3339(); + + let mut serializer = serde_json::Serializer::new(std::io::stdout()); + let mut output = serializer.serialize_seq(None).expect("Failed to write"); + + while let Some(directory) = roots.pop_back() { + // We're just going to keep going if we encounter errors reading a + // directory + let Ok(reader) = std::fs::read_dir(&directory) else { + continue; + }; + for entry in reader { + let Ok(entry) = entry else { continue }; + if entry.file_name() == "." || entry.file_name() == ".." { + continue; + } + let file = entry.path(); + let Ok(metadata) = entry.metadata() else { + continue; + }; + if metadata.is_dir() { + // Any child directories should be explored later + roots.push_back(file) + } else { + let Some(user) = users::get_user_by_uid(metadata.st_uid()) else { + continue; + }; + let Some(group) = users::get_group_by_gid(metadata.st_gid()) else { + continue; + }; + output + .serialize_element(&UnixFileData { + atime: to_seconds(metadata.accessed()), + ctime: to_seconds(metadata.created()), + fetched: &fetched, + file: file.to_str().expect("File name is not Unicode"), + group: group.name().to_str().unwrap_or(""), + host: &hostname, + mtime: to_seconds(metadata.modified()), + perms: metadata.permissions().mode(), + size: metadata.len(), + user: user.name().to_str().unwrap_or(""), + }) + .expect("Failed to write entry"); + } + } + } + output.end().expect("Failed to write"); +} diff --git a/json-dir-list/wix/main.wxs b/json-dir-list/wix/main.wxs new file mode 100644 index 000000000..6002ed34d --- /dev/null +++ b/json-dir-list/wix/main.wxs @@ -0,0 +1,228 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + + + + + + + + + + + + + + + diff --git a/shawk/Cargo.toml b/shawk/Cargo.toml index 690c10c69..0ff3d2318 100644 --- a/shawk/Cargo.toml +++ b/shawk/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2021" repository = "https://github.com/oicr-gsi/shesmu" authors = ["Andre Masella "] +description = "Shesmu command-line data extraction tool" [package.metadata.wix] upgrade-guid = "5ABBED1A-D1EE-446A-B684-E664CF1BD0CD" @@ -18,21 +19,3 @@ reqwest = { version = "^0.11", features = ["blocking", "json"] } rpassword = "^7.2" serde = { version = "^1.0", features = ["derive"] } serde_yaml = "^0.9" - -# The profile that 'cargo dist' will build with -[profile.dist] -inherits = "release" -lto = "thin" - -# Config for 'cargo dist' -[workspace.metadata.dist] -# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) -cargo-dist-version = "0.11.1" -# CI backends to support -ci = ["github"] -# The installers to generate for each app -installers = ["shell", "msi"] -# Target platforms to build apps for (Rust target-triple syntax) -targets = ["aarch64-apple-darwin", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] -# Publish jobs to run in CI -pr-run-mode = "plan" diff --git a/shawk/wix/main.wxs b/shawk/wix/main.wxs new file mode 100644 index 000000000..c5804f1af --- /dev/null +++ b/shawk/wix/main.wxs @@ -0,0 +1,228 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1 + 1 + + + + + + + + + + + + + + + + + +