From 028291e891fff7693f487e538a26ba0165959bd0 Mon Sep 17 00:00:00 2001 From: Tarek Elsayed <60650661+tareknaser@users.noreply.github.com> Date: Fri, 10 May 2024 19:29:57 +0300 Subject: [PATCH 1/5] `fs-storage`: format as valid JSON when write to disk (#41) Previously, the write to disk format for `FileStorage` was not a valid JSON. This commit fixes this. Other changes include: - The version is bumped to 3. - Field `data` of `FileStorageData` is renamed to `entries` - Support for reading version 2 plaintext `FileStorage` format for backwards compatibility is added. - This support is provided via a helper function `read_version_2_fs`. - A unit test for `read_version_2_fs` is also added. --------- Signed-off-by: Tarek --- fs-storage/src/file_storage.rs | 197 ++++++++++++++++++++------------- fs-storage/src/lib.rs | 1 + fs-storage/src/utils.rs | 82 ++++++++++++++ 3 files changed, 201 insertions(+), 79 deletions(-) create mode 100644 fs-storage/src/utils.rs diff --git a/fs-storage/src/file_storage.rs b/fs-storage/src/file_storage.rs index 23efeab5..d9096400 100644 --- a/fs-storage/src/file_storage.rs +++ b/fs-storage/src/file_storage.rs @@ -1,5 +1,6 @@ +use serde::{Deserialize, Serialize}; use std::fs::{self, File}; -use std::io::{BufRead, BufReader, BufWriter, Write}; +use std::io::{BufWriter, Write}; use std::time::SystemTime; use std::{ collections::BTreeMap, @@ -7,22 +8,56 @@ use std::{ }; use crate::base_storage::BaseStorage; +use crate::utils::read_version_2_fs; use data_error::{ArklibError, Result}; -const STORAGE_VERSION: i32 = 2; -const STORAGE_VERSION_PREFIX: &str = "version "; +/* +Note on `FileStorage` Versioning: -pub struct FileStorage { +`FileStorage` is a basic key-value storage system that persists data to disk. + +In version 2, `FileStorage` stored data in a plaintext format. +Starting from version 3, data is stored in JSON format. + +For backward compatibility, we provide a helper function `read_version_2_fs` to read version 2 format. +*/ +const STORAGE_VERSION: i32 = 3; + +/// Represents a file storage system that persists data to disk. +pub struct FileStorage +where + K: Ord, +{ label: String, path: PathBuf, timestamp: SystemTime, - data: BTreeMap, + data: FileStorageData, +} + +/// A struct that represents the data stored in a [`FileStorage`] instance. +/// +/// +/// This is the data that is serialized and deserialized to and from disk. +#[derive(Serialize, Deserialize)] +pub struct FileStorageData +where + K: Ord, +{ + version: i32, + entries: BTreeMap, } impl FileStorage where - K: Ord + Clone + serde::Serialize + serde::de::DeserializeOwned, - V: Clone + serde::Serialize + serde::de::DeserializeOwned, + K: Ord + + Clone + + serde::Serialize + + serde::de::DeserializeOwned + + std::str::FromStr, + V: Clone + + serde::Serialize + + serde::de::DeserializeOwned + + std::str::FromStr, { /// Create a new file storage with a diagnostic label and file path pub fn new(label: String, path: &Path) -> Self { @@ -30,63 +65,44 @@ where label, path: PathBuf::from(path), timestamp: SystemTime::now(), - data: BTreeMap::new(), + data: FileStorageData { + version: STORAGE_VERSION, + entries: BTreeMap::new(), + }, }; // Load the data from the file - file_storage.data = match file_storage.read_fs() { + file_storage.data.entries = match file_storage.read_fs() { Ok(data) => data, Err(_) => BTreeMap::new(), }; file_storage } - - /// Verify the version stored in the file header - fn verify_version(&self, header: &str) -> Result<()> { - if !header.starts_with(STORAGE_VERSION_PREFIX) { - return Err(ArklibError::Storage( - self.label.clone(), - "Unknown storage version prefix".to_owned(), - )); - } - - let version = header[STORAGE_VERSION_PREFIX.len()..] - .parse::() - .map_err(|_err| { - ArklibError::Storage( - self.label.clone(), - "Failed to parse storage version".to_owned(), - ) - })?; - - if version != STORAGE_VERSION { - return Err(ArklibError::Storage( - self.label.clone(), - format!( - "Storage version mismatch: expected {}, found {}", - STORAGE_VERSION, version - ), - )); - } - - Ok(()) - } } impl BaseStorage for FileStorage where - K: Ord + Clone + serde::Serialize + serde::de::DeserializeOwned, - V: Clone + serde::Serialize + serde::de::DeserializeOwned, + K: Ord + + Clone + + serde::Serialize + + serde::de::DeserializeOwned + + std::str::FromStr, + V: Clone + + serde::Serialize + + serde::de::DeserializeOwned + + std::str::FromStr, { - fn set(&mut self, id: K, value: V) { - self.data.insert(id, value); + /// Set a key-value pair in the storage + fn set(&mut self, key: K, value: V) { + self.data.entries.insert(key, value); self.timestamp = std::time::SystemTime::now(); self.write_fs() .expect("Failed to write data to disk"); } + /// Remove a key-value pair from the storage given a key fn remove(&mut self, id: &K) -> Result<()> { - self.data.remove(id).ok_or_else(|| { + self.data.entries.remove(id).ok_or_else(|| { ArklibError::Storage(self.label.clone(), "Key not found".to_owned()) })?; self.timestamp = std::time::SystemTime::now(); @@ -95,6 +111,8 @@ where Ok(()) } + /// Compare the timestamp of the storage file with the timestamp of the storage instance + /// to determine if the storage file has been updated. fn is_storage_updated(&self) -> Result { let file_timestamp = fs::metadata(&self.path)?.modified()?; let file_time_secs = file_timestamp @@ -109,35 +127,59 @@ where Ok(file_time_secs > self_time_secs) } + /// Read the data from the storage file fn read_fs(&mut self) -> Result> { - let file = fs::File::open(&self.path)?; - let reader = BufReader::new(file); - let mut lines = reader.lines(); + if !self.path.exists() { + return Err(ArklibError::Storage( + self.label.clone(), + "File does not exist".to_owned(), + )); + } - let new_timestamp = fs::metadata(&self.path)?.modified()?; - match lines.next() { - Some(header) => { - let header = header?; - self.verify_version(&header)?; - let mut data = String::new(); - for line in lines { - let line = line?; - if line.is_empty() { - continue; - } - data.push_str(&line); + // First check if the file starts with "version: 2" + let file_content = std::fs::read_to_string(&self.path)?; + if file_content.starts_with("version: 2") { + // Attempt to parse the file using the legacy version 2 storage format of FileStorage. + match read_version_2_fs(&self.path) { + Ok(data) => { + log::info!( + "Version 2 storage format detected for {}", + self.label + ); + self.timestamp = fs::metadata(&self.path)?.modified()?; + return Ok(data); + } + Err(_) => { + return Err(ArklibError::Storage( + self.label.clone(), + "Storage seems to be version 2, but failed to parse" + .to_owned(), + )); } - let data: BTreeMap = serde_json::from_str(&data)?; - self.timestamp = new_timestamp; - Ok(data) - } - None => Err(ArklibError::Storage( + }; + } + + let file = fs::File::open(&self.path)?; + let data: FileStorageData = serde_json::from_reader(file) + .map_err(|err| { + ArklibError::Storage(self.label.clone(), err.to_string()) + })?; + let version = data.version; + if version != STORAGE_VERSION { + return Err(ArklibError::Storage( self.label.clone(), - "Storage file is missing header".to_owned(), - )), + format!( + "Storage version mismatch: expected {}, got {}", + STORAGE_VERSION, version + ), + )); } + self.timestamp = fs::metadata(&self.path)?.modified()?; + + Ok(data.entries) } + /// Write the data to the storage file fn write_fs(&mut self) -> Result<()> { let parent_dir = self.path.parent().ok_or_else(|| { ArklibError::Storage( @@ -148,30 +190,24 @@ where fs::create_dir_all(parent_dir)?; let file = File::create(&self.path)?; let mut writer = BufWriter::new(file); - - writer.write_all( - format!("{}{}\n", STORAGE_VERSION_PREFIX, STORAGE_VERSION) - .as_bytes(), - )?; - - let value_map = self.data.clone(); - let value_data = serde_json::to_string(&value_map)?; + let value_data = serde_json::to_string_pretty(&self.data)?; writer.write_all(value_data.as_bytes())?; let new_timestamp = fs::metadata(&self.path)?.modified()?; if new_timestamp == self.timestamp { - return Err("Timestamp didn't update".into()); + return Err("Timestamp has not been updated".into()); } self.timestamp = new_timestamp; log::info!( "{} {} entries have been written", self.label, - value_map.len() + self.data.entries.len() ); Ok(()) } + /// Erase the storage file from disk fn erase(&self) -> Result<()> { fs::remove_file(&self.path).map_err(|err| { ArklibError::Storage(self.label.clone(), err.to_string()) @@ -179,9 +215,12 @@ where } } -impl AsRef> for FileStorage { +impl AsRef> for FileStorage +where + K: Ord, +{ fn as_ref(&self) -> &BTreeMap { - &self.data + &self.data.entries } } diff --git a/fs-storage/src/lib.rs b/fs-storage/src/lib.rs index 3b81ebfe..a6a6b03c 100644 --- a/fs-storage/src/lib.rs +++ b/fs-storage/src/lib.rs @@ -1,5 +1,6 @@ pub mod base_storage; pub mod file_storage; +mod utils; pub const ARK_FOLDER: &str = ".ark"; // Should not be lost if possible diff --git a/fs-storage/src/utils.rs b/fs-storage/src/utils.rs new file mode 100644 index 00000000..b5b6830a --- /dev/null +++ b/fs-storage/src/utils.rs @@ -0,0 +1,82 @@ +use data_error::Result; +use std::collections::BTreeMap; +use std::path::Path; + +/// Parses version 2 `FileStorage` format and returns the data as a BTreeMap +/// +/// Version 2 `FileStorage` format represents data as a BTreeMap in plaintext. +/// +/// For example: +/// ```text +/// version: 2 +/// key1:1 +/// key2:2 +/// key3:3 +/// ``` +pub fn read_version_2_fs(path: &Path) -> Result> +where + K: Ord + + Clone + + serde::Serialize + + serde::de::DeserializeOwned + + std::str::FromStr, + V: Clone + + serde::Serialize + + serde::de::DeserializeOwned + + std::str::FromStr, +{ + // First check if the file starts with "version: 2" + let file_content = std::fs::read_to_string(path)?; + if !file_content.starts_with("version: 2") { + return Err(data_error::ArklibError::Parse); + } + + // Parse the file content into a BTreeMap + let mut data = BTreeMap::new(); + for line in file_content.lines().skip(1) { + let mut parts = line.split(':'); + let key = parts + .next() + .unwrap() + .parse() + .map_err(|_| data_error::ArklibError::Parse)?; + let value = parts + .next() + .unwrap() + .parse() + .map_err(|_| data_error::ArklibError::Parse)?; + + data.insert(key, value); + } + + Ok(data) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempdir::TempDir; + + /// Test reading a legacy version 2 `FileStorage` file + #[test] + fn test_read_legacy_fs() { + let temp_dir = TempDir::new("ark-rust").unwrap(); + let file_path = temp_dir.path().join("test_read_legacy_fs"); + let file_content = r#"version: 2 +key1:1 +key2:2 +key3:3 +"#; + let mut file = std::fs::File::create(&file_path).unwrap(); + file.write_all(file_content.as_bytes()).unwrap(); + + // Read the file and check the data + let data: BTreeMap = + read_version_2_fs(&file_path).unwrap(); + assert_eq!(data.len(), 3); + assert_eq!(data.get("key1"), Some(&1)); + assert_eq!(data.get("key2"), Some(&2)); + assert_eq!(data.get("key3"), Some(&3)); + } +} From c87108b8bed886190e3e10c6395febf40d16060f Mon Sep 17 00:00:00 2001 From: Tarek Elsayed <60650661+tareknaser@users.noreply.github.com> Date: Sat, 11 May 2024 07:32:35 +0300 Subject: [PATCH 2/5] #40: Add a Note for Benchmarks' Profiles in `README` (#33) * feat(docs): add a note for benchmarks' profiles in `README` Signed-off-by: Tarek * feat(docs): add a note about cargo flamegraph on macos limitations Signed-off-by: Tarek --------- Signed-off-by: Tarek --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index 073ee50b..3b98deda 100644 --- a/README.md +++ b/README.md @@ -70,3 +70,29 @@ Our benchmark suite includes tests on local files and directories. These benchma You have the flexibility to benchmark specific files or folders by modifying the variables within the benchmark files. By default, the benchmarks operate on the [`test-assets/`](test-assets/) directory and its contents. You can change the directory/files by setting the `DIR_PATH` and `FILE_PATHS` variables to the desired values. For pre-benchmark assessment of required time to index a huge local folder, you can modify `test_build_resource_index` test case in `src/index.rs`. + +### Generating Profiles for Benchmarks + +[flamegraph](https://github.com/flamegraph-rs/flamegraph) is a Cargo command that uses perf/DTrace to profile your code and then displays the results in a flame graph. It works on Linux and all platforms that support DTrace (macOS, FreeBSD, NetBSD, and possibly Windows). + +To install `flamegraph`, run: + +```bash +cargo install flamegraph +``` + +To generate a flame graph for `index_build_benchmark`, use the following command: + +```bash +cargo flamegraph --bench index_build_benchmark -o index_build_benchmark.svg -- --bench +``` + +> [!NOTE] +> Running `cargo flamegraph` on macOS requires `DTrace`. MacOS System Integrity Protection (`SIP`) is enabled by default, which restricts most uses of `dtrace`. +> +> To work around this issue, you have two options: +> +> - Boot into recovery mode and disable some SIP protections. +> - Run as superuser to enable DTrace. This can be achieved by using `cargo flamegraph --root ...`. +> +> For further details, please refer to https://github.com/flamegraph-rs/flamegraph?tab=readme-ov-file#dtrace-on-macos From babddd9c4b9319497e4bef7fbed196042efbfdf4 Mon Sep 17 00:00:00 2001 From: Tarek Elsayed <60650661+tareknaser@users.noreply.github.com> Date: Sat, 11 May 2024 07:33:32 +0300 Subject: [PATCH 3/5] Include Windows and macOS Targets for CI (#39) * feat(data-pdf): fix `libpdfium` path in build script for macOS Signed-off-by: Tarek * fix(windows): remove dependency on `pprof`, incompatible with windows `pprof` was only used for profiling benchmarks which is not critical. This was causing issues for building `ark-rust` on windows Signed-off-by: Tarek * feat(ci): add macOS and windows jobs to github actions Both run with each commit Signed-off-by: Tarek * feat(ci): add a github workflow for binary releases We're holding off on adding a macOS ARM check to our build workflow because it involves expenses. Testing it for every PR or commit might not be worth it, so we'll include it in the release workflow instead. Since we haven't set up a release workflow yet, we'll treat it as a reminder to run it when we do release builds. Signed-off-by: Tarek * feat(docs): add logo and CI badges to README Signed-off-by: Tarek * feat(ci): add main branch workflows to run on all machines Signed-off-by: Tarek * feat(ci): add mac intel to main branch ci Signed-off-by: Tarek --------- Signed-off-by: Tarek --- .github/workflows/build.yml | 36 +++++++++---- .github/workflows/main_branch.yml | 54 +++++++++++++++++++ README.md | 10 +++- data-pdf/build.rs | 2 +- data-resource/Cargo.toml | 1 - .../benches/compute_bytes_benchmark.rs | 3 +- fs-index/Cargo.toml | 1 - fs-index/benches/index_build_benchmark.rs | 3 +- logo.svg | 1 + 9 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/main_branch.yml create mode 100644 logo.svg diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 94800057..1401fef1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,31 +6,47 @@ env: CARGO_TERM_COLOR: always jobs: - check: - name: Build ArkLib + linux: + name: Build on Linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy - + components: rustfmt, clippy - name: Check run: cargo check - - name: Format run: | cargo fmt --all -- --check - cargo clippy - + cargo clippy - name: Build Debug run: cargo build --verbose - - name: Run tests run: cargo test --verbose - - name: Build Release run: cargo build --verbose --release + + windows: + name: Test on Windows + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Run tests + run: cargo test --workspace --verbose + + mac-intel: + name: Test on macOS Intel + runs-on: macos-11 + + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Run tests + run: cargo test --workspace --verbose diff --git a/.github/workflows/main_branch.yml b/.github/workflows/main_branch.yml new file mode 100644 index 00000000..b7aa126b --- /dev/null +++ b/.github/workflows/main_branch.yml @@ -0,0 +1,54 @@ +name: Main Branch Workflow + +on: + push: + branches: + - main + schedule: + # Run each week + - cron: "0 0 * * 0" + +jobs: + linux: + name: Linux + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Run tests + run: cargo test --workspace --verbose --release + + windows: + name: Windows + runs-on: windows-latest + + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Run tests + run: cargo test --workspace --verbose --release + + mac-intel: + name: MacOS Intel + runs-on: macos-11 + + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Run tests + run: cargo test --workspace --verbose --release + + mac-arm: + name: MacOS ARM + runs-on: macos-13-xlarge + + steps: + - uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + - name: Run tests + run: cargo test --workspace --verbose --release diff --git a/README.md b/README.md index 3b98deda..5cd05fef 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,14 @@ # ArkLib -This is the home of core ARK library. +
+ ARK Logo + +[![License](https://img.shields.io/github/license/ARK-Builders/ark-rust.svg)](LICENSE) +[![Build Status](https://github.com/ARK-Builders/ark-rust/actions/workflows/build.yml/badge.svg)](https://github.com/ARK-Builders/ark-rust/actions/workflows/build.yml) + +**The home of core ARK library** + +
Being implemented in Rust, it provides us capability to port all our apps to all common platforms. Right now, Android is supported by using the [arklib-android](https://github.com/arK-Builders/arklib-android) project. And for Linux/macOS/Windows, the library can be used as-is and easily embedded into an app, e.g. built with [Tauri](https://tauri.app/). Development docs will come sometime. diff --git a/data-pdf/build.rs b/data-pdf/build.rs index 7e869175..f74315ac 100644 --- a/data-pdf/build.rs +++ b/data-pdf/build.rs @@ -89,7 +89,7 @@ fn main() { } | OperatingSystem::Darwin => fs_extra::file::move_file( PathBuf::from(&out_dir) - .join("bin") + .join("lib") .join("libpdfium.dylib"), PathBuf::from(&out_dir).join("libpdfium.dylib"), &CopyOptions::new(), diff --git a/data-resource/Cargo.toml b/data-resource/Cargo.toml index bbd9292f..cd3c8ab1 100644 --- a/data-resource/Cargo.toml +++ b/data-resource/Cargo.toml @@ -21,7 +21,6 @@ anyhow = "1" [dev-dependencies] # benchmarking criterion = { version = "0.5", features = ["html_reports"] } -pprof = { version = "0.13", features = ["criterion", "flamegraph"] } rand = "0.8" [[bench]] diff --git a/data-resource/benches/compute_bytes_benchmark.rs b/data-resource/benches/compute_bytes_benchmark.rs index fccd0b89..c715a490 100644 --- a/data-resource/benches/compute_bytes_benchmark.rs +++ b/data-resource/benches/compute_bytes_benchmark.rs @@ -1,6 +1,5 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use data_resource::ResourceId; -use pprof::criterion::{Output, PProfProfiler}; use rand::prelude::*; use std::fs; @@ -66,7 +65,7 @@ fn compute_bytes_on_files_benchmark(c: &mut Criterion) { criterion_group!( name = benches; - config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + config = Criterion::default(); targets = compute_bytes_on_raw_data, compute_bytes_on_files_benchmark ); criterion_main!(benches); diff --git a/fs-index/Cargo.toml b/fs-index/Cargo.toml index b7284d44..d43037bd 100644 --- a/fs-index/Cargo.toml +++ b/fs-index/Cargo.toml @@ -28,7 +28,6 @@ tempdir = "0.3.7" rstest = '0.18.2' # benchmarking criterion = { version = "0.5", features = ["html_reports"] } -pprof = { version = "0.13", features = ["criterion", "flamegraph"] } rand = "0.8" [[bench]] diff --git a/fs-index/benches/index_build_benchmark.rs b/fs-index/benches/index_build_benchmark.rs index dcdb7b2d..2d1abb7f 100644 --- a/fs-index/benches/index_build_benchmark.rs +++ b/fs-index/benches/index_build_benchmark.rs @@ -2,7 +2,6 @@ use criterion::{ black_box, criterion_group, criterion_main, BenchmarkId, Criterion, }; use fs_index::index::ResourceIndex; -use pprof::criterion::{Output, PProfProfiler}; const DIR_PATH: &str = "../test-assets/"; // Set the path to the directory containing the resources here @@ -36,7 +35,7 @@ fn index_build_benchmark(c: &mut Criterion) { criterion_group! { name = benches; - config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + config = Criterion::default(); targets = index_build_benchmark } criterion_main!(benches); diff --git a/logo.svg b/logo.svg new file mode 100644 index 00000000..09a5fe8d --- /dev/null +++ b/logo.svg @@ -0,0 +1 @@ + Ark-Logo-Only From 9fb321d592da05759ee4153cef2affe7e30c9b91 Mon Sep 17 00:00:00 2001 From: Kirill Taran Date: Sat, 11 May 2024 15:49:39 +0700 Subject: [PATCH 4/5] Update README --- README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5cd05fef..d2d7b7e9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ArkLib +# ARK
ARK Logo @@ -6,17 +6,24 @@ [![License](https://img.shields.io/github/license/ARK-Builders/ark-rust.svg)](LICENSE) [![Build Status](https://github.com/ARK-Builders/ark-rust/actions/workflows/build.yml/badge.svg)](https://github.com/ARK-Builders/ark-rust/actions/workflows/build.yml) -**The home of core ARK library** +**The home of the ARK framework**
-Being implemented in Rust, it provides us capability to port all our apps to all common platforms. Right now, Android is supported by using the [arklib-android](https://github.com/arK-Builders/arklib-android) project. And for Linux/macOS/Windows, the library can be used as-is and easily embedded into an app, e.g. built with [Tauri](https://tauri.app/). Development docs will come sometime. +Being implemented in Rust, it provides us capability to port any apps using ARK to all common platforms. Right now, Android is supported by using the [ark-android](https://github.com/ARK-Builders/ark-android) project. And for Linux/macOS/Windows, the framework can be used as-is and easily embedded into an app, e.g. built with [Tauri](https://tauri.app/). -**The Concept of the library** +Development docs will come sometime. -The purpose of the library is to manage _resource index_ of folders with various _user data_, as well as to manage user-defined metadata: tags, scores, arbitrary properties like movie title or description. Such a metadata is persisted to filesystem for easier sync and backup. The _resource index_ provides us with [content addressing](https://en.wikipedia.org/wiki/Content-addressable_storage) and allows easier storage and versions tracking. We also believe it'll allow easier cross-device sync implementation. +**The Concept of the Framework** -## Packages +The framework is supposed to help in solving the following problems: +1. Management of user data, stored as simple files on the disk, as well as various kinds of metadata: tags, scores, arbitrary properties like movie title or description. Such a metadata is persisted to filesystem for easier sync, backup and migration by any 3rd-party tool. +2. Sync of both user data and metadata, across all user devices in P2P fashion. Cache syncing might be implemented later, too. +3. Version tracking for user data: not only text-files, but also images, videos etc. + +The core crate is `fs-index` which provides us with [content addressing](https://en.wikipedia.org/wiki/Content-addressable_storage) and allows easier storage and versions tracking. Another important crate is `fs-storage` which has the purpose of storing and transactional access to different kinds of metadata in the hidden `.ark` folder. Don't miss the `ark-cli` which is a Swiss Army Knife for flexible usage in scripts consuming data from ARK-enabled folders. + +## Crates
From 3f862d59631edc4e6a22129629598f0b69232ea0 Mon Sep 17 00:00:00 2001 From: Kirill Taran Date: Sat, 11 May 2024 22:18:20 +0700 Subject: [PATCH 5/5] Remove accidentally committed test data --- ark-cli/ark-shelf/16-720383087 | 1 - ark-cli/ark-shelf/18-1909444406 | 1 - 2 files changed, 2 deletions(-) delete mode 100644 ark-cli/ark-shelf/16-720383087 delete mode 100644 ark-cli/ark-shelf/18-1909444406 diff --git a/ark-cli/ark-shelf/16-720383087 b/ark-cli/ark-shelf/16-720383087 deleted file mode 100644 index 4b23f03e..00000000 --- a/ark-cli/ark-shelf/16-720383087 +++ /dev/null @@ -1 +0,0 @@ -http://bing.com/ \ No newline at end of file diff --git a/ark-cli/ark-shelf/18-1909444406 b/ark-cli/ark-shelf/18-1909444406 deleted file mode 100644 index d2a553a6..00000000 --- a/ark-cli/ark-shelf/18-1909444406 +++ /dev/null @@ -1 +0,0 @@ -http://google.com/ \ No newline at end of file