Merge branch 'main' into monoid_

.github/workflows/build.yml

@@ -6,31 +6,47 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  check:
-    name: Build ArkLib
+  linux:
+    name: Build on Linux
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v4
-
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
-            components: rustfmt, clippy
-
+          components: rustfmt, clippy
       - name: Check
         run: cargo check
-
       - name: Format
         run: |
           cargo fmt --all -- --check
-          cargo clippy 
-
+          cargo clippy
       - name: Build Debug
         run: cargo build --verbose
-
       - name: Run tests
         run: cargo test --verbose
-
       - name: Build Release
         run: cargo build --verbose --release
+
+  windows:
+    name: Test on Windows
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Run tests
+        run: cargo test --workspace --verbose
+
+  mac-intel:
+    name: Test on macOS Intel
+    runs-on: macos-11
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Run tests
+        run: cargo test --workspace --verbose

.github/workflows/main_branch.yml

@@ -0,0 +1,54 @@
+name: Main Branch Workflow
+
+on:
+  push:
+    branches:
+      - main
+  schedule:
+    # Run each week
+    - cron: "0 0 * * 0"
+
+jobs:
+  linux:
+    name: Linux
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Run tests
+        run: cargo test --workspace --verbose --release
+
+  windows:
+    name: Windows
+    runs-on: windows-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Run tests
+        run: cargo test --workspace --verbose --release
+
+  mac-intel:
+    name: MacOS Intel
+    runs-on: macos-11
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Run tests
+        run: cargo test --workspace --verbose --release
+
+  mac-arm:
+    name: MacOS ARM
+    runs-on: macos-13-xlarge
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+      - name: Run tests
+        run: cargo test --workspace --verbose --release

README.md

@@ -1,14 +1,29 @@
-# ArkLib
+# ARK
 
-This is the home of core ARK library.
+<div align="center">
+  <img src="logo.svg" alt="ARK Logo" width="75%" />
+
+[![License](https://img.shields.io/github/license/ARK-Builders/ark-rust.svg)](LICENSE)
+[![Build Status](https://github.com/ARK-Builders/ark-rust/actions/workflows/build.yml/badge.svg)](https://github.com/ARK-Builders/ark-rust/actions/workflows/build.yml)
+
+**The home of the ARK framework**
+
+</div>
 
-Being implemented in Rust, it provides us capability to port all our apps to all common platforms. Right now, Android is supported by using the [arklib-android](https://github.com/arK-Builders/arklib-android) project. And for Linux/macOS/Windows, the library can be used as-is and easily embedded into an app, e.g. built with [Tauri](https://tauri.app/). Development docs will come sometime.
+Being implemented in Rust, it provides us capability to port any apps using ARK to all common platforms. Right now, Android is supported by using the [ark-android](https://github.com/ARK-Builders/ark-android) project. And for Linux/macOS/Windows, the framework can be used as-is and easily embedded into an app, e.g. built with [Tauri](https://tauri.app/).
 
-**The Concept of the library**
+Development docs will come sometime.
 
-The purpose of the library is to manage _resource index_ of folders with various _user data_, as well as to manage user-defined metadata: tags, scores, arbitrary properties like movie title or description. Such a metadata is persisted to filesystem for easier sync and backup. The _resource index_ provides us with [content addressing](https://en.wikipedia.org/wiki/Content-addressable_storage) and allows easier storage and versions tracking. We also believe it'll allow easier cross-device sync implementation.
+**The Concept of the Framework**
 
-## Packages
+The framework is supposed to help in solving the following problems:
+1. Management of user data, stored as simple files on the disk, as well as various kinds of metadata: tags, scores, arbitrary properties like movie title or description. Such a metadata is persisted to filesystem for easier sync, backup and migration by any 3rd-party tool.
+2. Sync of both user data and metadata, across all user devices in P2P fashion. Cache syncing might be implemented later, too.
+3. Version tracking for user data: not only text-files, but also images, videos etc.
+
+The core crate is `fs-index` which provides us with [content addressing](https://en.wikipedia.org/wiki/Content-addressable_storage) and allows easier storage and versions tracking. Another important crate is `fs-storage` which has the purpose of storing and transactional access to different kinds of metadata in the hidden `.ark` folder. Don't miss the `ark-cli` which is a Swiss Army Knife for flexible usage in scripts consuming data from ARK-enabled folders.
+
+## Crates
 
 <div align="center">
 
@@ -70,3 +85,29 @@ Our benchmark suite includes tests on local files and directories. These benchma
 You have the flexibility to benchmark specific files or folders by modifying the variables within the benchmark files. By default, the benchmarks operate on the [`test-assets/`](test-assets/) directory and its contents. You can change the directory/files by setting the `DIR_PATH` and `FILE_PATHS` variables to the desired values.
 
 For pre-benchmark assessment of required time to index a huge local folder, you can modify `test_build_resource_index` test case in `src/index.rs`.
+
+### Generating Profiles for Benchmarks
+
+[flamegraph](https://github.com/flamegraph-rs/flamegraph) is a Cargo command that uses perf/DTrace to profile your code and then displays the results in a flame graph. It works on Linux and all platforms that support DTrace (macOS, FreeBSD, NetBSD, and possibly Windows).
+
+To install `flamegraph`, run:
+
+```bash
+cargo install flamegraph
+```
+
+To generate a flame graph for `index_build_benchmark`, use the following command:
+
+```bash
+cargo flamegraph --bench index_build_benchmark -o index_build_benchmark.svg -- --bench
+```
+
+> [!NOTE]
+> Running `cargo flamegraph` on macOS requires `DTrace`. MacOS System Integrity Protection (`SIP`) is enabled by default, which restricts most uses of `dtrace`.
+>
+> To work around this issue, you have two options:
+>
+> - Boot into recovery mode and disable some SIP protections.
+> - Run as superuser to enable DTrace. This can be achieved by using `cargo flamegraph --root ...`.
+>
+> For further details, please refer to https://github.com/flamegraph-rs/flamegraph?tab=readme-ov-file#dtrace-on-macos

data-pdf/build.rs

@@ -89,7 +89,7 @@ fn main() {
         }
         | OperatingSystem::Darwin => fs_extra::file::move_file(
             PathBuf::from(&out_dir)
-                .join("bin")
+                .join("lib")
                 .join("libpdfium.dylib"),
             PathBuf::from(&out_dir).join("libpdfium.dylib"),
             &CopyOptions::new(),

data-resource/Cargo.toml

@@ -21,7 +21,6 @@ anyhow = "1"
 [dev-dependencies]
 # benchmarking
 criterion = { version = "0.5", features = ["html_reports"] }
-pprof = { version = "0.13", features = ["criterion", "flamegraph"] }
 rand = "0.8"
 
 [[bench]]

data-resource/benches/compute_bytes_benchmark.rs

@@ -1,6 +1,5 @@
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use data_resource::ResourceId;
-use pprof::criterion::{Output, PProfProfiler};
 use rand::prelude::*;
 use std::fs;
 
@@ -66,7 +65,7 @@ fn compute_bytes_on_files_benchmark(c: &mut Criterion) {
 
 criterion_group!(
     name = benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    config = Criterion::default();
     targets = compute_bytes_on_raw_data, compute_bytes_on_files_benchmark
 );
 criterion_main!(benches);

fs-index/Cargo.toml

@@ -28,7 +28,6 @@ tempdir = "0.3.7"
 rstest = '0.18.2'
 # benchmarking
 criterion = { version = "0.5", features = ["html_reports"] }
-pprof = { version = "0.13", features = ["criterion", "flamegraph"] }
 rand = "0.8"
 
 [[bench]]

fs-index/benches/index_build_benchmark.rs

@@ -2,7 +2,6 @@ use criterion::{
     black_box, criterion_group, criterion_main, BenchmarkId, Criterion,
 };
 use fs_index::index::ResourceIndex;
-use pprof::criterion::{Output, PProfProfiler};
 
 const DIR_PATH: &str = "../test-assets/"; // Set the path to the directory containing the resources here
 
@@ -36,7 +35,7 @@ fn index_build_benchmark(c: &mut Criterion) {
 
 criterion_group! {
     name = benches;
-    config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+    config = Criterion::default();
     targets = index_build_benchmark
 }
 criterion_main!(benches);