Skip to content

Commit

Permalink
Add ReadDir(Files)Nth API call, which allows skipping files (useful f…
Browse files Browse the repository at this point in the history 
…or resuming iteration)
  • Loading branch information
@sosthene-nitrokey
sosthene-nitrokey committed Apr 6, 2023
1 parent e052671 commit 3807bf2
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 8 deletions.
19 changes: 19 additions & 0 deletions src/api.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ generate_enums! {
Hash: 12
// TODO: add ReadDir{First,Next}, not loading data, if needed for efficiency
ReadDirFilesFirst: 13
ReadDirFilesNth: 31
ReadDirFilesNext: 14
ReadFile: 15
Metadata: 26
Expand All @@ -63,6 +64,7 @@ generate_enums! {

// // CreateDir, <-- implied by WriteFile
ReadDirFirst: 31 // <-- gets Option<FileType> to restrict to just dir/file DirEntries,
ReadDirNth: 31
ReadDirNext: 32 // <-- gets Option<FileType> to restrict to just dir/file DirEntries,
// // returns simplified Metadata
// // ReadDirFilesFirst: 23 // <-- returns contents
Expand Down Expand Up @@ -229,13 +231,24 @@ pub mod request {
- dir: PathBuf
- user_attribute: Option<UserAttribute>

ReadDirFilesNth:
- location: Location
- dir: PathBuf
- start_at: usize
- user_attribute: Option<UserAttribute>

ReadDirFilesNext:

ReadDirFirst:
- location: Location
- dir: PathBuf
- not_before_filename: Option<PathBuf>

ReadDirNth:
- location: Location
- dir: PathBuf
- start_at: usize

ReadDirNext:

ReadFile:
Expand Down Expand Up @@ -419,12 +432,18 @@ pub mod reply {
ReadDirFilesFirst:
- data: Option<Message>

ReadDirFilesNth:
- data: Option<Message>

ReadDirFilesNext:
- data: Option<Message>

ReadDirFirst:
- entry: Option<DirEntry>

ReadDirNth:
- data: Option<Message>

ReadDirNext:
- entry: Option<DirEntry>

Expand Down
28 changes: 28 additions & 0 deletions src/service.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,20 @@ impl<P: Platform> ServiceResources<P> {
};
Ok(Reply::ReadDirFirst(reply::ReadDirFirst { entry: maybe_entry } ))
}
Request::ReadDirNth(request) => {
let maybe_entry = match filestore.read_dir_nth(&request.dir, request.location, request.start_at)? {
Some((entry, read_dir_state)) => {
ctx.read_dir_state = Some(read_dir_state);
Some(entry)
}
None => {
ctx.read_dir_state = None;
None

}
};
Ok(Reply::ReadDirFirst(reply::ReadDirFirst { entry: maybe_entry } ))
}

Request::ReadDirNext(_request) => {
// ensure next call has nothing to work with, unless we store state again
Expand Down Expand Up @@ -391,6 +405,20 @@ impl<P: Platform> ServiceResources<P> {
Ok(Reply::ReadDirFilesFirst(reply::ReadDirFilesFirst { data: maybe_data } ))
}

Request::ReadDirFilesNth(request) => {
let maybe_data = match filestore.read_dir_files_nth(&request.dir, request.location, request.start_at, request.user_attribute.clone())? {
Some((data, state)) => {
ctx.read_dir_files_state = Some(state);
data
}
None => {
ctx.read_dir_files_state = None;
None
}
};
Ok(Reply::ReadDirFilesFirst(reply::ReadDirFilesFirst { data: maybe_data } ))
}

Request::ReadDirFilesNext(_request) => {
let read_dir_files_state = ctx.read_dir_files_state.take();

Expand Down
101 changes: 93 additions & 8 deletions src/store/filestore.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,19 @@ pub trait Filestore {
filename: PathBuf,
) -> Result<Option<PathBuf>>;

/// Iterate over entries of a directory (both file and directory entries).
///
/// This function is modeled after `std::fs::read_dir`, within the limitations of our setup.
///
/// In case an entry was found, the returned option also contains state, so the expected
/// call to `read_dir_next` can resume operation.
fn read_dir_nth(
&mut self,
dir: &PathBuf,
location: Location,
start_at: usize,
) -> Result<Option<(DirEntry, ReadDirState)>>;

/// Iterate over entries of a directory (both file and directory entries).
///
/// This function is modeled after `std::fs::read_dir`, within the limitations of our setup.
Expand Down Expand Up @@ -125,6 +138,21 @@ pub trait Filestore {
user_attribute: Option<UserAttribute>,
) -> Result<Option<(Option<Message>, ReadDirFilesState)>>;

/// Iterate over contents of files inside a directory.
///
/// This has no equivalent in `std::fs`, it is an optimization to avoid duplicate
/// calls and a more complicated state machine (interspersing read_dir_first/next calls
/// with some sort of "fetch data").
///
/// Additionally, files may optionally be filtered via attributes.
fn read_dir_files_nth(
&mut self,
clients_dir: &PathBuf,
location: Location,
start_at: usize,
user_attribute: Option<UserAttribute>,
) -> Result<Option<(Option<Message>, ReadDirFilesState)>>;

/// Continuation of `read_dir_files_first`.
fn read_dir_files_next(
&mut self,
Expand All @@ -139,6 +167,7 @@ impl<S: Store> ClientFilestore<S> {
clients_dir: &PathBuf,
location: Location,
not_before: Option<&PathBuf>,
start_at: usize,
fs: &'static Fs<F>,
) -> Result<Option<(DirEntry, ReadDirState)>> {
let dir = self.actual_path(clients_dir)?;
Expand All @@ -148,7 +177,7 @@ impl<S: Store> ClientFilestore<S> {
// this is an iterator with Item = (usize, Result<DirEntry>)
(&mut it)
// skip over `.` and `..`
.skip(2)
.skip(2 + start_at)
// todo: try ?-ing out of this (the API matches std::fs, where read/write errors
// can occur during operation)
//
Expand Down Expand Up @@ -232,6 +261,7 @@ impl<S: Store> ClientFilestore<S> {
clients_dir: &PathBuf,
location: Location,
user_attribute: Option<UserAttribute>,
start_at: usize,
fs: &'static Fs<F>,
) -> Result<Option<(Option<Message>, ReadDirFilesState)>> {
let dir = self.actual_path(clients_dir)?;
Expand All @@ -247,8 +277,7 @@ impl<S: Store> ClientFilestore<S> {
.map(Result::unwrap)
// skip over directories (including `.` and `..`)
.filter(|entry| entry.file_type().is_file())
// take first entry that meets requirements
.find(|entry| {
.filter(|entry| {
if let Some(user_attribute) = user_attribute.as_ref() {
let mut path = dir.clone();
path.push(entry.file_name());
Expand All @@ -257,14 +286,16 @@ impl<S: Store> ClientFilestore<S> {
.unwrap();

if let Some(attribute) = attribute {
user_attribute == attribute.data()
user_attribute != attribute.data()
} else {
false
true
}
} else {
true
}
})
// take nth entry that meets requirements
.nth(start_at)
// if there is an entry, construct the state that needs storing out of it,
// and return the file's contents.
// the client, and return both the entry and the state
Expand Down Expand Up @@ -403,13 +434,32 @@ impl<S: Store> Filestore for ClientFilestore<S> {
) -> Result<Option<(DirEntry, ReadDirState)>> {
match location {
Location::Internal => {
self.read_dir_first_impl(clients_dir, location, not_before, self.store.ifs())
self.read_dir_first_impl(clients_dir, location, not_before, 0, self.store.ifs())
}
Location::External => {
self.read_dir_first_impl(clients_dir, location, not_before, 0, self.store.efs())
}
Location::Volatile => {
self.read_dir_first_impl(clients_dir, location, not_before, 0, self.store.vfs())
}
}
}

fn read_dir_nth(
&mut self,
clients_dir: &PathBuf,
location: Location,
start_at: usize,
) -> Result<Option<(DirEntry, ReadDirState)>> {
match location {
Location::Internal => {
self.read_dir_first_impl(clients_dir, location, None, start_at, self.store.ifs())
}
Location::External => {
self.read_dir_first_impl(clients_dir, location, not_before, self.store.efs())
self.read_dir_first_impl(clients_dir, location, None, start_at, self.store.efs())
}
Location::Volatile => {
self.read_dir_first_impl(clients_dir, location, not_before, self.store.vfs())
self.read_dir_first_impl(clients_dir, location, None, start_at, self.store.vfs())
}
}
}
Expand All @@ -433,18 +483,53 @@ impl<S: Store> Filestore for ClientFilestore<S> {
clients_dir,
location,
user_attribute,
0,
self.store.ifs(),
),
Location::External => self.read_dir_files_first_impl(
clients_dir,
location,
user_attribute,
0,
self.store.efs(),
),
Location::Volatile => self.read_dir_files_first_impl(
clients_dir,
location,
user_attribute,
0,
self.store.vfs(),
),
}
}

fn read_dir_files_nth(
&mut self,
clients_dir: &PathBuf,
location: Location,
start_at: usize,
user_attribute: Option<UserAttribute>,
) -> Result<Option<(Option<Message>, ReadDirFilesState)>> {
match location {
Location::Internal => self.read_dir_files_first_impl(
clients_dir,
location,
user_attribute,
start_at,
self.store.ifs(),
),
Location::External => self.read_dir_files_first_impl(
clients_dir,
location,
user_attribute,
start_at,
self.store.efs(),
),
Location::Volatile => self.read_dir_files_first_impl(
clients_dir,
location,
user_attribute,
start_at,
self.store.vfs(),
),
}
Expand Down

0 comments on commit 3807bf2

Please sign in to comment.