From d2f7ccc486c9c26cd9652ea1f537dbe92d87c0ee Mon Sep 17 00:00:00 2001 From: sharkAndshar Date: Thu, 26 Dec 2024 16:30:07 +0800 Subject: [PATCH 1/8] Scan *.tiff either --- martin/src/args/root.rs | 36 +++++++++++++----- martin/src/config.rs | 6 +-- martin/src/file_config.rs | 36 +++++++++++------- tests/config.yaml | 2 +- tests/expected/auto/save_config.yaml | 2 +- tests/expected/configured/save_config.yaml | 2 +- ...rgba_u8_nodata.tif => rgba_u8_nodata.tiff} | Bin 7 files changed, 54 insertions(+), 30 deletions(-) rename tests/fixtures/cog/{rgba_u8_nodata.tif => rgba_u8_nodata.tiff} (100%) diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 040d1e05f..57ed13789 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -99,17 +99,17 @@ impl Args { #[cfg(feature = "pmtiles")] if !cli_strings.is_empty() { - config.pmtiles = parse_file_args(&mut cli_strings, "pmtiles", true); + config.pmtiles = parse_file_args(&mut cli_strings, &["pmtiles"], true); } #[cfg(feature = "mbtiles")] if !cli_strings.is_empty() { - config.mbtiles = parse_file_args(&mut cli_strings, "mbtiles", false); + config.mbtiles = parse_file_args(&mut cli_strings, &["mbtiles"], false); } #[cfg(feature = "cog")] if !cli_strings.is_empty() { - config.cog = parse_file_args(&mut cli_strings, "tif", false); + config.cog = parse_file_args(&mut cli_strings, &["tif", "tiff"], false); } #[cfg(feature = "sprites")] @@ -125,13 +125,13 @@ impl Args { } } -#[cfg(any(feature = "pmtiles", feature = "mbtiles"))] -fn is_url(s: &str, extension: &str) -> bool { +#[cfg(any(feature = "pmtiles", feature = "mbtiles", feature = "cog"))] +fn is_url(s: &str, extension: &[&str]) -> bool { if s.starts_with("http") { if let Ok(url) = url::Url::parse(s) { if url.scheme() == "http" || url.scheme() == "https" { if let Some(ext) = url.path().rsplit('.').next() { - return ext == extension; + return extension.contains(&ext); } } } @@ -139,21 +139,26 @@ fn is_url(s: &str, extension: &str) -> bool { false } -#[cfg(any(feature = "pmtiles", feature = "mbtiles"))] +#[cfg(any(feature = "pmtiles", feature = "mbtiles", feature = "cog"))] pub fn parse_file_args( cli_strings: &mut Arguments, - extension: &str, + extensions: &[&str], allow_url: bool, ) -> FileConfigEnum { use crate::args::State::{Ignore, Share, Take}; let paths = cli_strings.process(|s| { let path = PathBuf::from(s); - if allow_url && is_url(s, extension) { + if allow_url && is_url(s, extensions) { Take(path) } else if path.is_dir() { Share(path) - } else if path.is_file() && path.extension().map_or(false, |e| e == extension) { + } else if path.is_file() + && extensions.iter().any(|&expected_ext| { + path.extension() + .is_some_and(|actual_ext| actual_ext == expected_ext) + }) + { Take(path) } else { Ignore @@ -275,4 +280,15 @@ mod tests { let bad = vec!["foobar".to_string()]; assert!(matches!(err, UnrecognizableConnections(v) if v == bad)); } + + #[test] + fn cli_multiple_extensions() { + let args = Args::parse_from(["martin", "../tests/fixtures/cog"]); + + let env = FauxEnv::default(); + let mut config = Config::default(); + let err = args.merge_into_config(&mut config, &env); + println!("{:?}", err); + println!("{:?}", config); + } } diff --git a/martin/src/config.rs b/martin/src/config.rs index 4e9ca02f8..0715e08b1 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -178,21 +178,21 @@ impl Config { #[cfg(feature = "pmtiles")] if !self.pmtiles.is_empty() { let cfg = &mut self.pmtiles; - let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "pmtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), &["pmtiles"]); sources.push(Box::pin(val)); } #[cfg(feature = "mbtiles")] if !self.mbtiles.is_empty() { let cfg = &mut self.mbtiles; - let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "mbtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), &["mbtiles"]); sources.push(Box::pin(val)); } #[cfg(feature = "cog")] if !self.cog.is_empty() { let cfg = &mut self.cog; - let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "tif"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), &["tif", "tiff"]); sources.push(Box::pin(val)); } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 1196c84fe..08f62e00b 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -235,7 +235,7 @@ pub async fn resolve_files( config: &mut FileConfigEnum, idr: &IdResolver, cache: OptMainCache, - extension: &str, + extension: &[&str], ) -> MartinResult { resolve_int(config, idr, cache, extension) .map_err(crate::MartinError::from) @@ -246,7 +246,7 @@ async fn resolve_int( config: &mut FileConfigEnum, idr: &IdResolver, cache: OptMainCache, - extension: &str, + extension: &[&str], ) -> FileResult { let Some(cfg) = config.extract_file_config(cache)? else { return Ok(TileInfoSources::default()); @@ -285,16 +285,20 @@ async fn resolve_int( for path in cfg.paths { if let Some(url) = parse_url(T::parse_urls(), &path)? { - let id = url - .path_segments() - .and_then(Iterator::last) - .and_then(|s| { - // Strip extension and trailing dot, or keep the original string - s.strip_suffix(extension) - .and_then(|s| s.strip_suffix('.')) - .or(Some(s)) - }) - .unwrap_or("pmt_web_source"); + let target_ext = extension.iter().find(|&e| url.to_string().ends_with(e)); + let id = if let Some(ext) = target_ext { + url.path_segments() + .and_then(Iterator::last) + .and_then(|s| { + // Strip extension and trailing dot, or keep the original string + s.strip_suffix(ext) + .and_then(|s| s.strip_suffix('.')) + .or(Some(s)) + }) + .unwrap_or("web_source") + } else { + "web_source" + }; let id = idr.resolve(id, url.to_string()); configs.insert(id.clone(), FileConfigSrc::Path(path)); @@ -337,13 +341,17 @@ async fn resolve_int( Ok(results) } -fn dir_to_paths(path: &Path, extension: &str) -> Result, FileError> { +fn dir_to_paths(path: &Path, extension: &[&str]) -> Result, FileError> { Ok(path .read_dir() .map_err(|e| IoError(e, path.to_path_buf()))? .filter_map(Result::ok) .filter(|f| { - f.path().extension().filter(|e| *e == extension).is_some() && f.path().is_file() + f.path() + .extension() + .filter(|e| extension.iter().any(|x| x == e)) + .is_some() + && f.path().is_file() }) .map(|f| f.path()) .collect()) diff --git a/tests/config.yaml b/tests/config.yaml index d963453da..b7343e958 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -182,7 +182,7 @@ sprites: cog: paths: - - tests/fixtures/cog/rgba_u8_nodata.tif + - tests/fixtures/cog/rgba_u8_nodata.tiff sources: cog-src1: tests/fixtures/cog/rgba_u8.tif cog-src2: tests/fixtures/cog/rgb_u8.tif diff --git a/tests/expected/auto/save_config.yaml b/tests/expected/auto/save_config.yaml index 9878a45c9..dfa59a7f2 100644 --- a/tests/expected/auto/save_config.yaml +++ b/tests/expected/auto/save_config.yaml @@ -260,7 +260,7 @@ cog: sources: rgb_u8: tests/fixtures/cog/rgb_u8.tif rgba_u8: tests/fixtures/cog/rgba_u8.tif - rgba_u8_nodata: tests/fixtures/cog/rgba_u8_nodata.tif + rgba_u8_nodata: tests/fixtures/cog/rgba_u8_nodata.tiff sprites: tests/fixtures/sprites/src1 fonts: - tests/fixtures/fonts/overpass-mono-regular.ttf diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index e3d665f0e..fa6723f96 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -170,7 +170,7 @@ cog: sources: cog-src1: tests/fixtures/cog/rgba_u8.tif cog-src2: tests/fixtures/cog/rgb_u8.tif - rgba_u8_nodata: tests/fixtures/cog/rgba_u8_nodata.tif + rgba_u8_nodata: tests/fixtures/cog/rgba_u8_nodata.tiff sprites: paths: tests/fixtures/sprites/src1 sources: diff --git a/tests/fixtures/cog/rgba_u8_nodata.tif b/tests/fixtures/cog/rgba_u8_nodata.tiff similarity index 100% rename from tests/fixtures/cog/rgba_u8_nodata.tif rename to tests/fixtures/cog/rgba_u8_nodata.tiff From 7df3a73604a918a4c3da5f6a48dcb26495c88259 Mon Sep 17 00:00:00 2001 From: sharkAndshar Date: Thu, 26 Dec 2024 16:52:37 +0800 Subject: [PATCH 2/8] Update doc --- docs/src/config-file.md | 2 +- docs/src/sources-cog-files.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 0749063e3..be5f16e31 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -202,7 +202,7 @@ mbtiles: # Cloud Optimized GeoTIFF File Sources cog: paths: - # scan this whole dir, matching all *.tif files + # scan this whole dir, matching all *.tif and *.tiff files - /dir-path # specific TIFF file will be published as a cog source - /path/to/cogfile.tif diff --git a/docs/src/sources-cog-files.md b/docs/src/sources-cog-files.md index 70ee4b9f7..55d3deace 100644 --- a/docs/src/sources-cog-files.md +++ b/docs/src/sources-cog-files.md @@ -22,10 +22,10 @@ Martin can also serve raster sources like local [COG(Cloud Optimized GeoTIFF)](h ```bash # Configured with a directory containing TIFF files. martin /with/tiff/dir1 /with/tiff/dir2 -# Configured with dedicated TIFF file. -martin /path/to/target1.tif /path/to/target1.tif +# Configured with dedicated TIFF file, `.tiff` works either +martin /path/to/target1.tif /path/to/target2.tiff # Configured with a combination of directories and dedicated TIFF files. -martin /with/tiff/files /path/to/target.tif +martin /with/tiff/files /path/to/target1.tif /path/to/target2.tiff ``` ## Run Martin with configuration file @@ -46,11 +46,11 @@ cache_size_mb: 8 cog: paths: - # scan this whole dir, matching all *.tif files + # scan this whole dir, matching all *.tif and *.tiff files - /dir-path # specific TIFF file will be published as a cog source - /path/to/target1.tif - - /path/to/target2.tif + - /path/to/target2.tiff sources: # named source matching source name to a single file cog-src1: /path/to/cog1.tif From 67f2bd8a8ee224d843e8bd29c1bd996723391919 Mon Sep 17 00:00:00 2001 From: sharkAndshar Date: Fri, 27 Dec 2024 14:30:48 +0800 Subject: [PATCH 3/8] update doc --- docs/src/sources-cog-files.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/sources-cog-files.md b/docs/src/sources-cog-files.md index 55d3deace..f9966ebe3 100644 --- a/docs/src/sources-cog-files.md +++ b/docs/src/sources-cog-files.md @@ -20,9 +20,9 @@ Martin can also serve raster sources like local [COG(Cloud Optimized GeoTIFF)](h ## Run Martin with CLI to serve cog files ```bash -# Configured with a directory containing TIFF files. +# Configured with a directory containing `*.tif` or `*.tiff` TIFF files. martin /with/tiff/dir1 /with/tiff/dir2 -# Configured with dedicated TIFF file, `.tiff` works either +# Configured with dedicated TIFF file martin /path/to/target1.tif /path/to/target2.tiff # Configured with a combination of directories and dedicated TIFF files. martin /with/tiff/files /path/to/target1.tif /path/to/target2.tiff From 8d79503dfdac3951b1bfb822053aca60428f09c3 Mon Sep 17 00:00:00 2001 From: sharkAndshar Date: Fri, 27 Dec 2024 14:30:54 +0800 Subject: [PATCH 4/8] update test --- martin/src/args/root.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 57ed13789..444798333 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -171,6 +171,8 @@ pub fn parse_file_args( #[cfg(test)] mod tests { + use insta::{assert_snapshot, assert_yaml_snapshot}; + use super::*; use crate::args::PreferredEncoding; use crate::test_utils::FauxEnv; @@ -283,12 +285,22 @@ mod tests { #[test] fn cli_multiple_extensions() { - let args = Args::parse_from(["martin", "../tests/fixtures/cog"]); + let args = Args::parse_from([ + "martin", + "../tests/fixtures/cog/rgb_u8.tif", + "../tests/fixtures/cog/rgba_u8_nodata.tiff", + "../tests/fixtures/cog/rgba_u8.tif", + ]); let env = FauxEnv::default(); let mut config = Config::default(); let err = args.merge_into_config(&mut config, &env); - println!("{:?}", err); - println!("{:?}", config); + assert!(err.is_ok()); + assert_yaml_snapshot!(config, @r#" + cog: + - "../tests/fixtures/cog/rgb_u8.tif" + - "../tests/fixtures/cog/rgba_u8_nodata.tiff" + - "../tests/fixtures/cog/rgba_u8.tif" + "#); } } From 9b15a6d5f5dbb398787fc7a7ac89184a341b43e6 Mon Sep 17 00:00:00 2001 From: sharkAndshar Date: Fri, 27 Dec 2024 14:44:41 +0800 Subject: [PATCH 5/8] clippy --- martin/src/args/root.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 444798333..7fa6e5ebc 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -171,7 +171,7 @@ pub fn parse_file_args( #[cfg(test)] mod tests { - use insta::{assert_snapshot, assert_yaml_snapshot}; + use insta::assert_yaml_snapshot; use super::*; use crate::args::PreferredEncoding; From 2515c777754916a668d3d004249a9da68ab45919 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 28 Dec 2024 19:53:07 +0800 Subject: [PATCH 6/8] update doc --- docs/src/config-file.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/src/config-file.md b/docs/src/config-file.md index be5f16e31..fe09dc476 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -205,7 +205,8 @@ cog: # scan this whole dir, matching all *.tif and *.tiff files - /dir-path # specific TIFF file will be published as a cog source - - /path/to/cogfile.tif + - /path/to/cogfile1.tif + - /path/to/cogfile2.tiff sources: # named source matching source name to a single file cog-src1: /path/to/cog1.tif From b1f40d3966f0b16639c0e54acb6d05350a4e99a6 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 28 Dec 2024 21:16:37 +0800 Subject: [PATCH 7/8] update test --- martin/src/args/root.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 7fa6e5ebc..00f3075c6 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -287,7 +287,7 @@ mod tests { fn cli_multiple_extensions() { let args = Args::parse_from([ "martin", - "../tests/fixtures/cog/rgb_u8.tif", + "../tests/fixtures/cog", "../tests/fixtures/cog/rgba_u8_nodata.tiff", "../tests/fixtures/cog/rgba_u8.tif", ]); From 7a14e4da3fd8df22b98db9e49daa4d7910f8d719 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 28 Dec 2024 21:50:09 +0800 Subject: [PATCH 8/8] cleanup --- martin/src/file_config.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 08f62e00b..e1c54702a 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -349,7 +349,11 @@ fn dir_to_paths(path: &Path, extension: &[&str]) -> Result, FileErr .filter(|f| { f.path() .extension() - .filter(|e| extension.iter().any(|x| x == e)) + .filter(|actual_ext| { + extension + .iter() + .any(|expected_ext| expected_ext == actual_ext) + }) .is_some() && f.path().is_file() })