From a89fa33958495d7c7a2c412f2009a8e3c9b0ee1a Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Nov 2023 00:04:20 -0500 Subject: [PATCH 01/19] Use Rust 1.74 std::io::Error::other fn --- Cargo.toml | 2 +- martin/src/mbtiles/mod.rs | 7 +------ martin/src/pmtiles/mod.rs | 14 ++------------ 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 033c2a8fc..fa540a7fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = ["martin", "martin-tile-utils", "mbtiles"] edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/maplibre/martin" -rust-version = "1.70" +rust-version = "1.74" readme = "README.md" homepage = "https://martin.maplibre.org/" diff --git a/martin/src/mbtiles/mod.rs b/martin/src/mbtiles/mod.rs index 3029fed04..d9ea2dfcd 100644 --- a/martin/src/mbtiles/mod.rs +++ b/martin/src/mbtiles/mod.rs @@ -63,12 +63,7 @@ impl MbtSource { async fn new(id: String, path: PathBuf) -> FileResult { let mbt = MbtilesPool::new(&path) .await - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("{e:?}: Cannot open file {}", path.display()), - ) - }) + .map_err(|e| io::Error::other(format!("{e:?}: Cannot open file {}", path.display()))) .map_err(|e| IoError(e, path.clone()))?; let meta = mbt diff --git a/martin/src/pmtiles/mod.rs b/martin/src/pmtiles/mod.rs index ec6514531..9fd8c456f 100644 --- a/martin/src/pmtiles/mod.rs +++ b/martin/src/pmtiles/mod.rs @@ -318,22 +318,12 @@ impl PmtFileSource { pub async fn new(cache: PmtCache, id: String, path: PathBuf) -> FileResult { let backend = MmapBackend::try_from(path.as_path()) .await - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("{e:?}: Cannot open file {}", path.display()), - ) - }) + .map_err(|e| io::Error::other(format!("{e:?}: Cannot open file {}", path.display()))) .map_err(|e| IoError(e, path.clone()))?; let reader = AsyncPmTilesReader::try_from_cached_source(backend, cache).await; let reader = reader - .map_err(|e| { - io::Error::new( - io::ErrorKind::Other, - format!("{e:?}: Cannot open file {}", path.display()), - ) - }) + .map_err(|e| io::Error::other(format!("{e:?}: Cannot open file {}", path.display()))) .map_err(|e| IoError(e, path.clone()))?; Self::new_int(id, path, reader).await From 23f0e8820ded147609a002e3e4629cfb46e2263d Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Dec 2023 12:47:23 -0500 Subject: [PATCH 02/19] Deprecate few things, lock and lints (#1111) * Remove `CA_ROOT_FILE` env var support (has been deprecated for a while) * Remove `WATCH_MODE` env var warning - has not been in use since a long time ago * minor lints/clippy suggestions, lock update --- Cargo.lock | 28 ++++++++-------- docs/src/env-vars.md | 14 ++++---- martin/src/args/pg.rs | 10 +----- martin/src/args/root.rs | 27 ++++++--------- mbtiles/src/validation.rs | 70 +++++++++++++++++++-------------------- 5 files changed, 68 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f12a25112..c939d7e31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,9 +214,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "getrandom", @@ -578,9 +578,9 @@ checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" [[package]] name = "cargo-platform" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34637b3140142bdf929fb439e8aa4ebad7651ebf7b1080b3930aa16ac1459ff" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" dependencies = [ "serde", ] @@ -1820,13 +1820,13 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2110,9 +2110,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memmap2" @@ -3474,9 +3474,9 @@ checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "similar" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aeaf503862c419d66959f5d7ca015337d864e9c49485d771b732e2a20453597" +checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21" [[package]] name = "simplecss" @@ -3982,15 +3982,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/docs/src/env-vars.md b/docs/src/env-vars.md index 9df5cffa5..0c9ef7536 100644 --- a/docs/src/env-vars.md +++ b/docs/src/env-vars.md @@ -2,10 +2,10 @@ You can also configure Martin using environment variables, but only if the configuration file is not used. See [configuration section](config-file.md) on how to use environment variables with config files. See also [SSL configuration](pg-connections.md#postgresql-ssl-connections) section below. -| Environment var
Config File key | Example | Description | -|------------------------------------------|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `DATABASE_URL`
`connection_string` | `postgresql://postgres@localhost/db` | Postgres database connection | -| `DEFAULT_SRID`
`default_srid` | `4326` | If a PostgreSQL table has a geometry column with SRID=0, use this value instead | -| `PGSSLCERT`
`ssl_cert` | `./postgresql.crt` | A file with a client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT) | -| `PGSSLKEY`
`ssl_key` | `./postgresql.key` | A file with the key for the client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY) | -| `PGSSLROOTCERT`
`ssl_root_cert` | `./root.crt` | A file with trusted root certificate(s). The file should contain a sequence of PEM-formatted CA certificates. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT)
This env var used to be called `CA_ROOT_FILE`, but support for it will be removed soon. | +| Environment var
Config File key | Example | Description | +|------------------------------------------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `DATABASE_URL`
`connection_string` | `postgresql://postgres@localhost/db` | Postgres database connection | +| `DEFAULT_SRID`
`default_srid` | `4326` | If a PostgreSQL table has a geometry column with SRID=0, use this value instead | +| `PGSSLCERT`
`ssl_cert` | `./postgresql.crt` | A file with a client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT) | +| `PGSSLKEY`
`ssl_key` | `./postgresql.key` | A file with the key for the client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY) | +| `PGSSLROOTCERT`
`ssl_root_cert` | `./root.crt` | A file with trusted root certificate(s). The file should contain a sequence of PEM-formatted CA certificates. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT) | diff --git a/martin/src/args/pg.rs b/martin/src/args/pg.rs index 7f29960cb..a0da3a005 100644 --- a/martin/src/args/pg.rs +++ b/martin/src/args/pg.rs @@ -105,7 +105,6 @@ impl PgArgs { } for v in &[ - "CA_ROOT_FILE", "DANGER_ACCEPT_INVALID_CERTS", "DATABASE_URL", "DEFAULT_SRID", @@ -167,13 +166,6 @@ impl PgArgs { if result.ssl_root_cert.is_none() { result.ssl_root_cert = Self::parse_env_var(env, "PGSSLROOTCERT", "root certificate(s)"); } - if result.ssl_root_cert.is_none() { - result.ssl_root_cert = Self::parse_env_var( - env, - "CA_ROOT_FILE", - "root certificate(s). This setting is obsolete, please use PGSSLROOTCERT instead", - ); - } result } @@ -255,7 +247,7 @@ mod tests { ("DATABASE_URL", os("postgres://localhost:5432")), ("DEFAULT_SRID", os("10")), ("DANGER_ACCEPT_INVALID_CERTS", os("1")), - ("CA_ROOT_FILE", os("file")), + ("PGSSLROOTCERT", os("file")), ] .into_iter() .collect(), diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 518964009..2fc83c2d5 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -65,14 +65,11 @@ impl Args { pub fn merge_into_config<'a>( self, config: &mut Config, - env: &impl Env<'a>, + #[allow(unused_variables)] env: &impl Env<'a>, ) -> MartinResult<()> { if self.meta.watch { warn!("The --watch flag is no longer supported, and will be ignored"); } - if env.has_unused_var("WATCH_MODE") { - warn!("The WATCH_MODE env variable is no longer supported, and will be ignored"); - } if self.meta.config.is_some() && !self.meta.connection.is_empty() { return Err(ConfigAndConnectionsError(self.meta.connection)); } @@ -138,19 +135,17 @@ pub fn parse_file_args( ) -> FileConfigEnum { use crate::args::State::{Ignore, Share, Take}; - let paths = cli_strings.process(|s| match PathBuf::try_from(s) { - Ok(v) => { - if allow_url && is_url(s, extension) { - Take(v) - } else if v.is_dir() { - Share(v) - } else if v.is_file() && v.extension().map_or(false, |e| e == extension) { - Take(v) - } else { - Ignore - } + let paths = cli_strings.process(|s| { + let path = PathBuf::from(s); + if allow_url && is_url(s, extension) { + Take(path) + } else if path.is_dir() { + Share(path) + } else if path.is_file() && path.extension().map_or(false, |e| e == extension) { + Take(path) + } else { + Ignore } - Err(_) => Ignore, }); FileConfigEnum::new(paths) diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index 34ce483a8..9680a9806 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -456,6 +456,41 @@ LIMIT 1;" } } +/// Compute the hash of the combined tiles in the mbtiles file tiles table/view. +/// This should work on all mbtiles files perf `MBTiles` specification. +pub async fn calc_agg_tiles_hash(conn: &mut T) -> MbtResult +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + debug!("Calculating agg_tiles_hash"); + let query = query( + // The md5_concat func will return NULL if there are no rows in the tiles table. + // For our use case, we will treat it as an empty string, and hash that. + // `tile_data` values must be stored as a blob per MBTiles spec + // `md5` functions will fail if the value is not text/blob/null + // + // Note that ORDER BY controls the output ordering, which is important for the hash value, + // and having it at the top level would not order values properly. + // See https://sqlite.org/forum/forumpost/228bb96e12a746ce + " +SELECT coalesce( + (SELECT md5_concat_hex( + cast(zoom_level AS text), + cast(tile_column AS text), + cast(tile_row AS text), + tile_data + ) + OVER (ORDER BY zoom_level, tile_column, tile_row ROWS + BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) + FROM tiles + LIMIT 1), + md5_hex('') +); +", + ); + Ok(query.fetch_one(conn).await?.get::(0)) +} + #[cfg(test)] pub(crate) mod tests { use super::*; @@ -499,38 +534,3 @@ pub(crate) mod tests { Ok(()) } } - -/// Compute the hash of the combined tiles in the mbtiles file tiles table/view. -/// This should work on all mbtiles files perf `MBTiles` specification. -pub async fn calc_agg_tiles_hash(conn: &mut T) -> MbtResult -where - for<'e> &'e mut T: SqliteExecutor<'e>, -{ - debug!("Calculating agg_tiles_hash"); - let query = query( - // The md5_concat func will return NULL if there are no rows in the tiles table. - // For our use case, we will treat it as an empty string, and hash that. - // `tile_data` values must be stored as a blob per MBTiles spec - // `md5` functions will fail if the value is not text/blob/null - // - // Note that ORDER BY controls the output ordering, which is important for the hash value, - // and having it at the top level would not order values properly. - // See https://sqlite.org/forum/forumpost/228bb96e12a746ce - " -SELECT coalesce( - (SELECT md5_concat_hex( - cast(zoom_level AS text), - cast(tile_column AS text), - cast(tile_row AS text), - tile_data - ) - OVER (ORDER BY zoom_level, tile_column, tile_row ROWS - BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) - FROM tiles - LIMIT 1), - md5_hex('') -); -", - ); - Ok(query.fetch_one(conn).await?.get::(0)) -} From 6ee2493994e719ed7af2f923c01b5941fd9fd1f3 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 30 Dec 2023 22:48:23 -0500 Subject: [PATCH 03/19] Implement tile caching (#1105) Add a top level config parameter -- the size of cache memory (in MB) to use for caching tiles and PMT directories, defaulting to 512, and 0 to disable. This also removes the `pmtiles.dir_cache_size_mb` parameter (it will be ignored, but will give a warning) ``` cache_size_mb: 512 ``` The new cache will contain all tiles as provided by the source. So if PostgreSQL returns a non-compressed tile, the cache will contain the uncompressed variant, and will be compressed for each response. This will be fixed in the later releases. Note that fonts and sprites are not cached at this time, and are still a TODO. --- Cargo.lock | 40 ++-- debian/config.yaml | 4 +- docs/src/config-file.md | 5 +- martin-tile-utils/Cargo.toml | 2 +- martin-tile-utils/src/lib.rs | 2 +- martin/Cargo.toml | 6 +- martin/benches/bench.rs | 5 +- martin/src/args/root.rs | 7 + martin/src/bin/martin-cp.rs | 45 ++-- martin/src/bin/martin.rs | 6 +- martin/src/config.rs | 41 +++- martin/src/file_config.rs | 21 +- martin/src/lib.rs | 2 +- martin/src/pmtiles/mod.rs | 53 ++--- martin/src/source.rs | 1 + martin/src/sprites/mod.rs | 2 +- martin/src/srv/fonts.rs | 0 martin/src/srv/mod.rs | 8 +- martin/src/srv/server.rs | 4 +- martin/src/srv/sprites.rs | 27 +-- martin/src/srv/tiles.rs | 249 ++++++++++++--------- martin/src/srv/tiles_info.rs | 0 martin/src/utils/cache.rs | 91 ++++++++ martin/src/utils/mod.rs | 3 + martin/src/utils/xyz.rs | 2 +- martin/tests/mb_server_test.rs | 1 + martin/tests/pg_server_test.rs | 2 + martin/tests/pmt_server_test.rs | 1 + martin/tests/utils/pg_utils.rs | 4 +- mbtiles/Cargo.toml | 2 +- tests/config.yaml | 4 +- tests/expected/configured/save_config.yaml | 2 +- 32 files changed, 417 insertions(+), 225 deletions(-) mode change 100755 => 100644 martin/src/srv/fonts.rs mode change 100644 => 100755 martin/src/srv/tiles.rs mode change 100644 => 100755 martin/src/srv/tiles_info.rs create mode 100755 martin/src/utils/cache.rs diff --git a/Cargo.lock b/Cargo.lock index c939d7e31..e789b0ffe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d19de80eff169429ac1e9f48fffb163916b448a44e8e046186232046d9e1f9" +checksum = "ca87830a3e3fb156dc96cfbd31cb620265dd053be734723f22b760d6cc3c3051" [[package]] name = "approx" @@ -389,9 +389,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" dependencies = [ "proc-macro2", "quote", @@ -1025,9 +1025,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -1707,9 +1707,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2010,7 +2010,7 @@ dependencies = [ [[package]] name = "martin" -version = "0.12.0" +version = "0.13.0" dependencies = [ "actix-cors", "actix-http", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" dependencies = [ "approx", "insta", @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" dependencies = [ "actix-rt", "anyhow", @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75cb1540fadbd5b8fbccc4dddad2734eba435053f725621c070711a14bb5f4b8" +checksum = "a293318316cf6478ec1ad2a21c49390a8d5b5eae9fab736467d93fbc0edc29c5" dependencies = [ "unicode-ident", ] @@ -4004,18 +4004,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a48fd946b02c0a526b2e9481c8e2a17755e47039164a86c4070446e3a4614d" +checksum = "b2cd5904763bad08ad5513ddbb12cf2ae273ca53fa9f68e843e236ec6dfccc09" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.52" +version = "1.0.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7fbe9b594d6568a6a1443250a7e67d80b74e1e96f6d1715e1e21cc1888291d3" +checksum = "3dcf4a824cce0aeacd6f38ae6f24234c8e80d68632338ebaa1443b5df9e29e19" dependencies = [ "proc-macro2", "quote", @@ -4674,11 +4674,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] diff --git a/debian/config.yaml b/debian/config.yaml index f44db6ad2..30b9ad27e 100644 --- a/debian/config.yaml +++ b/debian/config.yaml @@ -7,6 +7,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 512 + # see https://maplibre.org/martin/config-file.html # postgres: @@ -17,7 +20,6 @@ worker_processes: 8 # auto_bounds: skip # pmtiles: -# dir_cache_size_mb: 100 # paths: # - /dir-path # - /path/to/pmtiles.pmtiles diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 19cfec94f..542677de1 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -24,6 +24,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 1024 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string. You can use env vars too, for example: @@ -155,8 +158,6 @@ postgres: # Publish PMTiles files from local disk or proxy to a web server pmtiles: - # Memory (in MB) to use for caching PMTiles directories [default: 32, 0 to disable]] - dir_cache_size_mb: 100 paths: # scan this whole dir, matching all *.pmtiles files - /dir-path diff --git a/martin-tile-utils/Cargo.toml b/martin-tile-utils/Cargo.toml index 1f6899566..90d39ce7d 100644 --- a/martin-tile-utils/Cargo.toml +++ b/martin-tile-utils/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "martin-tile-utils" -version = "0.4.0" +version = "0.4.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "Utilites to help with map tile processing, such as type and compression detection. Used by the MapLibre's Martin tile server." keywords = ["maps", "tiles", "mvt", "tileserver"] diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index 029949ebe..9df4fd498 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -86,7 +86,7 @@ impl Display for Format { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] pub enum Encoding { /// Data is not compressed, but it can be Uncompressed = 0b0000_0000, diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 38405a0f2..9e81bec80 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -3,7 +3,7 @@ lints.workspace = true [package] name = "martin" # Once the release is published with the hash, update https://github.com/maplibre/homebrew-martin -version = "0.12.0" +version = "0.13.0" authors = ["Stepan Kuzmin ", "Yuri Astrakhan ", "MapLibre contributors"] description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support" keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"] @@ -62,7 +62,7 @@ harness = false default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"] fonts = ["dep:bit-set", "dep:pbf_font_tools"] mbtiles = [] -pmtiles = ["dep:moka"] +pmtiles = [] postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"] sprites = ["dep:spreet"] bless-tests = [] @@ -85,7 +85,7 @@ json-patch = { workspace = true, optional = true } log.workspace = true martin-tile-utils.workspace = true mbtiles.workspace = true -moka = { workspace = true, optional = true } +moka.workspace = true num_cpus.workspace = true pbf_font_tools = { workspace = true, optional = true } pmtiles.workspace = true diff --git a/martin/benches/bench.rs b/martin/benches/bench.rs index 77363284b..27f4904b2 100644 --- a/martin/benches/bench.rs +++ b/martin/benches/bench.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use criterion::async_executor::FuturesExecutor; use criterion::{criterion_group, criterion_main, Criterion}; -use martin::srv::get_tile_response; +use martin::srv::DynTileSource; use martin::{ CatalogSourceEntry, MartinResult, Source, TileCoord, TileData, TileSources, UrlQuery, }; @@ -58,7 +58,8 @@ impl Source for NullSource { } async fn process_tile(sources: &TileSources) { - get_tile_response(sources, TileCoord { z: 0, x: 0, y: 0 }, "null", "", None) + let src = DynTileSource::new(sources, "null", Some(0), "", None, None).unwrap(); + src.get_http_response(TileCoord { z: 0, x: 0, y: 0 }) .await .unwrap(); } diff --git a/martin/src/args/root.rs b/martin/src/args/root.rs index 2fc83c2d5..7fc81d1be 100644 --- a/martin/src/args/root.rs +++ b/martin/src/args/root.rs @@ -43,6 +43,9 @@ pub struct MetaArgs { /// By default, only print if sources are auto-detected. #[arg(long)] pub save_config: Option, + /// Main cache size (in MB) + #[arg(short = 'C', long)] + pub cache_size: Option, /// **Deprecated** Scan for new sources on sources list requests #[arg(short, long, hide = true)] pub watch: bool, @@ -74,6 +77,10 @@ impl Args { return Err(ConfigAndConnectionsError(self.meta.connection)); } + if self.meta.cache_size.is_some() { + config.cache_size_mb = self.meta.cache_size; + } + self.srv.merge_into_config(&mut config.srv); #[allow(unused_mut)] diff --git a/martin/src/bin/martin-cp.rs b/martin/src/bin/martin-cp.rs index 405d16bab..feecbfc95 100644 --- a/martin/src/bin/martin-cp.rs +++ b/martin/src/bin/martin-cp.rs @@ -12,10 +12,10 @@ use futures::stream::{self, StreamExt}; use futures::TryStreamExt; use log::{debug, error, info, log_enabled}; use martin::args::{Args, ExtraArgs, MetaArgs, OsEnv, SrvArgs}; -use martin::srv::{get_tile_content, merge_tilejson, RESERVED_KEYWORDS}; +use martin::srv::{merge_tilejson, DynTileSource}; use martin::{ - append_rect, read_config, Config, IdResolver, MartinError, MartinResult, ServerState, Source, - TileCoord, TileData, TileRect, + append_rect, read_config, Config, MartinError, MartinResult, ServerState, Source, TileCoord, + TileData, TileRect, }; use martin_tile_utils::{bbox_to_xyz, TileInfo}; use mbtiles::sqlx::SqliteConnection; @@ -144,7 +144,8 @@ async fn start(copy_args: CopierArgs) -> MartinCpResult<()> { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; @@ -274,9 +275,18 @@ fn iterate_tiles(tiles: Vec) -> impl Iterator { async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> { let output_file = &args.output_file; let concurrency = args.concurrency.unwrap_or(1); - let (sources, _use_url_query, info) = state.tiles.get_sources(args.source.as_str(), None)?; - let sources = sources.as_slice(); - let tile_info = sources.first().unwrap().get_tile_info(); + + let src = DynTileSource::new( + &state.tiles, + args.source.as_str(), + None, + args.url_query.as_deref().unwrap_or_default(), + Some(parse_encoding(args.encoding.as_str())?), + None, + )?; + // parallel async below uses move, so we must only use copyable types + let src = &src; + let (tx, mut rx) = channel::(500); let tiles = compute_tile_ranges(&args); let mbt = Mbtiles::new(output_file)?; @@ -288,30 +298,26 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> } else { CopyDuplicateMode::Override }; - let mbt_type = init_schema(&mbt, &mut conn, sources, tile_info, &args).await?; - let query = args.url_query.as_deref(); - let req = TestRequest::default() - .insert_header((ACCEPT_ENCODING, args.encoding.as_str())) - .finish(); - let accept_encoding = AcceptEncoding::parse(&req)?; - let encodings = Some(&accept_encoding); + let mbt_type = init_schema(&mbt, &mut conn, src.sources.as_slice(), src.info, &args).await?; let progress = Progress::new(&tiles); info!( - "Copying {} {tile_info} tiles from {} to {}", + "Copying {} {} tiles from {} to {}", progress.total, + src.info, args.source, args.output_file.display() ); try_join!( + // Note: for some reason, tests hang here without the `move` keyword async move { stream::iter(iterate_tiles(tiles)) .map(MartinResult::Ok) .try_for_each_concurrent(concurrency, |xyz| { let tx = tx.clone(); async move { - let tile = get_tile_content(sources, info, xyz, query, encodings).await?; + let tile = src.get_tile_content(xyz).await?; let data = tile.data; tx.send(TileXyz { xyz, data }) .await @@ -375,6 +381,13 @@ async fn run_tile_copy(args: CopyArgs, state: ServerState) -> MartinCpResult<()> Ok(()) } +fn parse_encoding(encoding: &str) -> MartinCpResult { + let req = TestRequest::default() + .insert_header((ACCEPT_ENCODING, encoding)) + .finish(); + Ok(AcceptEncoding::parse(&req)?) +} + async fn init_schema( mbt: &Mbtiles, conn: &mut SqliteConnection, diff --git a/martin/src/bin/martin.rs b/martin/src/bin/martin.rs index 5765e66e8..054970e49 100644 --- a/martin/src/bin/martin.rs +++ b/martin/src/bin/martin.rs @@ -4,8 +4,8 @@ use actix_web::dev::Server; use clap::Parser; use log::{error, info, log_enabled}; use martin::args::{Args, OsEnv}; -use martin::srv::{new_server, RESERVED_KEYWORDS}; -use martin::{read_config, Config, IdResolver, MartinResult}; +use martin::srv::new_server; +use martin::{read_config, Config, MartinResult}; const VERSION: &str = env!("CARGO_PKG_VERSION"); @@ -24,7 +24,7 @@ async fn start(args: Args) -> MartinResult { args.merge_into_config(&mut config, &env)?; config.finalize()?; - let sources = config.resolve(IdResolver::new(RESERVED_KEYWORDS)).await?; + let sources = config.resolve().await?; if let Some(file_name) = save_config { config.save_to_file(file_name)?; diff --git a/martin/src/config.rs b/martin/src/config.rs index f311d5e69..8fed9347f 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -18,13 +18,15 @@ use crate::fonts::FontSources; use crate::source::{TileInfoSources, TileSources}; #[cfg(feature = "sprites")] use crate::sprites::{SpriteConfig, SpriteSources}; -use crate::srv::SrvConfig; +use crate::srv::{SrvConfig, RESERVED_KEYWORDS}; +use crate::utils::{CacheValue, MainCache, OptMainCache}; use crate::MartinError::{ConfigLoadError, ConfigParseError, ConfigWriteError, NoSources}; use crate::{IdResolver, MartinResult, OptOneMany}; pub type UnrecognizedValues = HashMap; pub struct ServerState { + pub cache: OptMainCache, pub tiles: TileSources, #[cfg(feature = "sprites")] pub sprites: SpriteSources, @@ -32,8 +34,11 @@ pub struct ServerState { pub fonts: FontSources, } +#[serde_with::skip_serializing_none] #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Config { + pub cache_size_mb: Option, + #[serde(flatten)] pub srv: SrvConfig, @@ -107,19 +112,43 @@ impl Config { } } - pub async fn resolve(&mut self, idr: IdResolver) -> MartinResult { + pub async fn resolve(&mut self) -> MartinResult { + let resolver = IdResolver::new(RESERVED_KEYWORDS); + let cache_size = self.cache_size_mb.unwrap_or(512) * 1024 * 1024; + let cache = if cache_size > 0 { + info!("Initializing main cache with maximum size {cache_size}B"); + Some( + MainCache::builder() + .weigher(|_key, value: &CacheValue| -> u32 { + match value { + CacheValue::Tile(v) => v.len().try_into().unwrap_or(u32::MAX), + CacheValue::PmtDirectory(v) => { + v.get_approx_byte_size().try_into().unwrap_or(u32::MAX) + } + } + }) + .max_capacity(cache_size) + .build(), + ) + } else { + info!("Caching is disabled"); + None + }; + Ok(ServerState { - tiles: self.resolve_tile_sources(idr).await?, + tiles: self.resolve_tile_sources(&resolver, cache.clone()).await?, #[cfg(feature = "sprites")] sprites: SpriteSources::resolve(&mut self.sprites)?, #[cfg(feature = "fonts")] fonts: FontSources::resolve(&mut self.fonts)?, + cache, }) } async fn resolve_tile_sources( &mut self, - #[allow(unused_variables)] idr: IdResolver, + #[allow(unused_variables)] idr: &IdResolver, + #[allow(unused_variables)] cache: OptMainCache, ) -> MartinResult { #[allow(unused_mut)] let mut sources: Vec>>>> = @@ -133,14 +162,14 @@ impl Config { #[cfg(feature = "pmtiles")] if !self.pmtiles.is_empty() { let cfg = &mut self.pmtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "pmtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "pmtiles"); sources.push(Box::pin(val)); } #[cfg(feature = "mbtiles")] if !self.mbtiles.is_empty() { let cfg = &mut self.mbtiles; - let val = crate::file_config::resolve_files(cfg, idr.clone(), "mbtiles"); + let val = crate::file_config::resolve_files(cfg, idr, cache.clone(), "mbtiles"); sources.push(Box::pin(val)); } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 636ec3f1c..3692f75be 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -14,7 +14,7 @@ use crate::file_config::FileError::{ InvalidFilePath, InvalidSourceFilePath, InvalidSourceUrl, IoError, }; use crate::source::{Source, TileInfoSources}; -use crate::utils::{IdResolver, OptOneMany}; +use crate::utils::{IdResolver, OptMainCache, OptOneMany}; use crate::MartinResult; use crate::OptOneMany::{Many, One}; @@ -48,7 +48,7 @@ pub enum FileError { } pub trait ConfigExtras: Clone + Debug + Default + PartialEq + Send { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, _cache: OptMainCache) -> FileResult<()> { Ok(()) } @@ -127,7 +127,10 @@ impl FileConfigEnum { } } - pub fn extract_file_config(&mut self) -> FileResult>> { + pub fn extract_file_config( + &mut self, + cache: OptMainCache, + ) -> FileResult>> { let mut res = match self { FileConfigEnum::None => return Ok(None), FileConfigEnum::Path(path) => FileConfig { @@ -140,7 +143,7 @@ impl FileConfigEnum { }, FileConfigEnum::Config(cfg) => mem::take(cfg), }; - res.custom.init_parsing()?; + res.custom.init_parsing(cache)?; Ok(Some(res)) } @@ -218,20 +221,22 @@ pub struct FileConfigSource { pub async fn resolve_files( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> MartinResult { - resolve_int(config, idr, extension) + resolve_int(config, idr, cache, extension) .map_err(crate::MartinError::from) .await } async fn resolve_int( config: &mut FileConfigEnum, - idr: IdResolver, + idr: &IdResolver, + cache: OptMainCache, extension: &str, ) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(cache)? else { return Ok(TileInfoSources::default()); }; diff --git a/martin/src/lib.rs b/martin/src/lib.rs index c0c9d4728..ef4aad9f7 100644 --- a/martin/src/lib.rs +++ b/martin/src/lib.rs @@ -10,7 +10,7 @@ pub use source::{CatalogSourceEntry, Source, Tile, TileData, TileSources, UrlQue mod utils; pub use utils::{ append_rect, decode_brotli, decode_gzip, IdResolver, MartinError, MartinResult, OptBoolObj, - OptOneMany, TileCoord, TileRect, + OptOneMany, TileCoord, TileRect, NO_MAIN_CACHE, }; pub mod args; diff --git a/martin/src/pmtiles/mod.rs b/martin/src/pmtiles/mod.rs index 9fd8c456f..b24389a39 100644 --- a/martin/src/pmtiles/mod.rs +++ b/martin/src/pmtiles/mod.rs @@ -9,7 +9,6 @@ use std::sync::Arc; use async_trait::async_trait; use log::{trace, warn}; use martin_tile_utils::{Encoding, Format, TileInfo}; -use moka::future::Cache; use pmtiles::async_reader::AsyncPmTilesReader; use pmtiles::cache::{DirCacheResult, DirectoryCache}; use pmtiles::http::HttpBackend; @@ -24,20 +23,20 @@ use crate::config::UnrecognizedValues; use crate::file_config::FileError::{InvalidMetadata, InvalidUrlMetadata, IoError}; use crate::file_config::{ConfigExtras, FileError, FileResult, SourceConfigExtras}; use crate::source::UrlQuery; +use crate::utils::cache::get_cached_value; +use crate::utils::{CacheKey, CacheValue, OptMainCache}; use crate::{MartinResult, Source, TileCoord, TileData}; -type PmtCacheObject = Cache<(usize, usize), Directory>; - #[derive(Clone, Debug)] pub struct PmtCache { id: usize, - /// (id, offset) -> Directory, or None to disable caching - cache: Option, + /// Storing (id, offset) -> Directory, or None to disable caching + cache: OptMainCache, } impl PmtCache { #[must_use] - pub fn new(id: usize, cache: Option) -> Self { + pub fn new(id: usize, cache: OptMainCache) -> Self { Self { id, cache } } } @@ -45,17 +44,23 @@ impl PmtCache { #[async_trait] impl DirectoryCache for PmtCache { async fn get_dir_entry(&self, offset: usize, tile_id: u64) -> DirCacheResult { - if let Some(cache) = &self.cache { - if let Some(dir) = cache.get(&(self.id, offset)).await { - return dir.find_tile_id(tile_id).into(); - } + if let Some(dir) = get_cached_value!(&self.cache, CacheValue::PmtDirectory, { + CacheKey::PmtDirectory(self.id, offset) + }) { + dir.find_tile_id(tile_id).into() + } else { + DirCacheResult::NotCached } - DirCacheResult::NotCached } async fn insert_dir(&self, offset: usize, directory: Directory) { if let Some(cache) = &self.cache { - cache.insert((self.id, offset), directory).await; + cache + .insert( + CacheKey::PmtDirectory(self.id, offset), + CacheValue::PmtDirectory(directory), + ) + .await; } } } @@ -63,8 +68,6 @@ impl DirectoryCache for PmtCache { #[serde_with::skip_serializing_none] #[derive(Debug, Default, Serialize, Deserialize)] pub struct PmtConfig { - pub dir_cache_size_mb: Option, - #[serde(flatten)] pub unrecognized: UnrecognizedValues, @@ -78,12 +81,12 @@ pub struct PmtConfig { pub next_cache_id: AtomicUsize, #[serde(skip)] - pub cache: Option, + pub cache: OptMainCache, } impl PartialEq for PmtConfig { fn eq(&self, other: &Self) -> bool { - self.dir_cache_size_mb == other.dir_cache_size_mb && self.unrecognized == other.unrecognized + self.unrecognized == other.unrecognized } } @@ -91,7 +94,6 @@ impl Clone for PmtConfig { fn clone(&self) -> Self { // State is not shared between clones, only the serialized config Self { - dir_cache_size_mb: self.dir_cache_size_mb, unrecognized: self.unrecognized.clone(), ..Default::default() } @@ -107,24 +109,17 @@ impl PmtConfig { } impl ConfigExtras for PmtConfig { - fn init_parsing(&mut self) -> FileResult<()> { + fn init_parsing(&mut self, cache: OptMainCache) -> FileResult<()> { assert!(self.client.is_none()); assert!(self.cache.is_none()); self.client = Some(Client::new()); + self.cache = cache; - // Allow cache size to be disabled with 0 - let dir_cache_size = self.dir_cache_size_mb.unwrap_or(32) * 1024 * 1024; - if dir_cache_size > 0 { - self.cache = Some( - Cache::builder() - .weigher(|_key, value: &Directory| -> u32 { - value.get_approx_byte_size().try_into().unwrap_or(u32::MAX) - }) - .max_capacity(dir_cache_size) - .build(), - ); + if self.unrecognized.contains_key("dir_cache_size_mb") { + warn!("dir_cache_size_mb is no longer used. Instead, use cache_size_mb param in the root of the config file."); } + Ok(()) } diff --git a/martin/src/source.rs b/martin/src/source.rs index 01c2bc2b5..ba04293b9 100644 --- a/martin/src/source.rs +++ b/martin/src/source.rs @@ -167,6 +167,7 @@ mod tests { } } +#[derive(Debug, Clone)] pub struct Tile { pub data: TileData, pub info: TileInfo, diff --git a/martin/src/sprites/mod.rs b/martin/src/sprites/mod.rs index fa81f179b..7a0fc8cc4 100644 --- a/martin/src/sprites/mod.rs +++ b/martin/src/sprites/mod.rs @@ -77,7 +77,7 @@ pub struct SpriteSources(HashMap); impl SpriteSources { pub fn resolve(config: &mut FileConfigEnum) -> FileResult { - let Some(cfg) = config.extract_file_config()? else { + let Some(cfg) = config.extract_file_config(None)? else { return Ok(Self::default()); }; diff --git a/martin/src/srv/fonts.rs b/martin/src/srv/fonts.rs old mode 100755 new mode 100644 diff --git a/martin/src/srv/mod.rs b/martin/src/srv/mod.rs index ea94de2ea..b9ad026f8 100644 --- a/martin/src/srv/mod.rs +++ b/martin/src/srv/mod.rs @@ -1,14 +1,14 @@ mod config; pub use config::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT}; +#[cfg(feature = "fonts")] +mod fonts; + mod server; pub use server::{new_server, router, Catalog, RESERVED_KEYWORDS}; mod tiles; -pub use tiles::{get_tile_content, get_tile_response, TileRequest}; - -#[cfg(feature = "fonts")] -mod fonts; +pub use tiles::{DynTileSource, TileRequest}; mod tiles_info; pub use tiles_info::{merge_tilejson, SourceIDsRequest}; diff --git a/martin/src/srv/server.rs b/martin/src/srv/server.rs index 377638b66..4fe0e4d9d 100755 --- a/martin/src/srv/server.rs +++ b/martin/src/srv/server.rs @@ -112,7 +112,9 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server .allow_any_origin() .allowed_methods(vec!["GET"]); - let app = App::new().app_data(Data::new(state.tiles.clone())); + let app = App::new() + .app_data(Data::new(state.tiles.clone())) + .app_data(Data::new(state.cache.clone())); #[cfg(feature = "sprites")] let app = app.app_data(Data::new(state.sprites.clone())); diff --git a/martin/src/srv/sprites.rs b/martin/src/srv/sprites.rs index c3275bbeb..55a61a81b 100644 --- a/martin/src/srv/sprites.rs +++ b/martin/src/srv/sprites.rs @@ -4,28 +4,18 @@ use actix_web::error::ErrorNotFound; use actix_web::http::header::ContentType; use actix_web::web::{Data, Path}; use actix_web::{middleware, route, HttpResponse, Result as ActixResult}; +use spreet::Spritesheet; use crate::sprites::{SpriteError, SpriteSources}; use crate::srv::server::map_internal_error; use crate::srv::SourceIDsRequest; -pub fn map_sprite_error(e: SpriteError) -> actix_web::Error { - use SpriteError::SpriteNotFound; - match e { - SpriteNotFound(_) => ErrorNotFound(e.to_string()), - _ => map_internal_error(e), - } -} - #[route("/sprite/{source_ids}.png", method = "GET", method = "HEAD")] async fn get_sprite_png( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites - .get_sprites(&path.source_ids) - .await - .map_err(map_sprite_error)?; + let sheet = get_sprite(&path, &sprites).await?; Ok(HttpResponse::Ok() .content_type(ContentType::png()) .body(sheet.encode_png().map_err(map_internal_error)?)) @@ -41,9 +31,16 @@ async fn get_sprite_json( path: Path, sprites: Data, ) -> ActixResult { - let sheet = sprites + let sheet = get_sprite(&path, &sprites).await?; + Ok(HttpResponse::Ok().json(sheet.get_index())) +} + +async fn get_sprite(path: &SourceIDsRequest, sprites: &SpriteSources) -> ActixResult { + sprites .get_sprites(&path.source_ids) .await - .map_err(map_sprite_error)?; - Ok(HttpResponse::Ok().json(sheet.get_index())) + .map_err(|e| match e { + SpriteError::SpriteNotFound(_) => ErrorNotFound(e.to_string()), + _ => map_internal_error(e), + }) } diff --git a/martin/src/srv/tiles.rs b/martin/src/srv/tiles.rs old mode 100644 new mode 100755 index 32463a6f3..f25829966 --- a/martin/src/srv/tiles.rs +++ b/martin/src/srv/tiles.rs @@ -6,13 +6,18 @@ use actix_web::http::header::{ use actix_web::web::{Data, Path, Query}; use actix_web::{route, HttpMessage, HttpRequest, HttpResponse, Result as ActixResult}; use futures::future::try_join_all; +use log::trace; use martin_tile_utils::{Encoding, Format, TileInfo}; use serde::Deserialize; use crate::source::{Source, TileSources, UrlQuery}; use crate::srv::server::map_internal_error; -use crate::utils::{decode_brotli, decode_gzip, encode_brotli, encode_gzip}; -use crate::{Tile, TileCoord}; +use crate::utils::cache::get_or_insert_cached_value; +use crate::utils::{ + decode_brotli, decode_gzip, encode_brotli, encode_gzip, CacheKey, CacheValue, MainCache, + OptMainCache, +}; +use crate::{Tile, TileCoord, TileData}; static SUPPORTED_ENCODINGS: &[HeaderEnc] = &[ HeaderEnc::brotli(), @@ -33,125 +38,165 @@ async fn get_tile( req: HttpRequest, path: Path, sources: Data, + cache: Data, ) -> ActixResult { - let xyz = TileCoord { + let src = DynTileSource::new( + sources.as_ref(), + &path.source_ids, + Some(path.z), + req.query_string(), + req.get_header::(), + cache.as_ref().as_ref(), + )?; + + src.get_http_response(TileCoord { z: path.z, x: path.x, y: path.y, - }; - - let source_ids = &path.source_ids; - let query = req.query_string(); - let encodings = req.get_header::(); + }) + .await +} - get_tile_response(sources.as_ref(), xyz, source_ids, query, encodings).await +pub struct DynTileSource<'a> { + pub sources: Vec<&'a dyn Source>, + pub info: TileInfo, + pub query_str: Option<&'a str>, + pub query_obj: Option, + pub encodings: Option, + pub cache: Option<&'a MainCache>, } -pub async fn get_tile_response( - sources: &TileSources, - xyz: TileCoord, - source_ids: &str, - query: &str, - encodings: Option, -) -> ActixResult { - let (sources, use_url_query, info) = sources.get_sources(source_ids, Some(xyz.z))?; +impl<'a> DynTileSource<'a> { + pub fn new( + sources: &'a TileSources, + source_ids: &str, + zoom: Option, + query: &'a str, + encodings: Option, + cache: Option<&'a MainCache>, + ) -> ActixResult { + let (sources, use_url_query, info) = sources.get_sources(source_ids, zoom)?; - let query = use_url_query.then_some(query); - let tile = get_tile_content(sources.as_slice(), info, xyz, query, encodings.as_ref()).await?; + if sources.is_empty() { + return Err(ErrorNotFound("No valid sources found")); + } - Ok(if tile.data.is_empty() { - HttpResponse::NoContent().finish() - } else { - let mut response = HttpResponse::Ok(); - response.content_type(tile.info.format.content_type()); - if let Some(val) = tile.info.encoding.content_encoding() { - response.insert_header((CONTENT_ENCODING, val)); + let mut query_obj = None; + let mut query_str = None; + if use_url_query && !query.is_empty() { + query_obj = Some(Query::::from_query(query)?.into_inner()); + query_str = Some(query); } - response.body(tile.data) - }) -} -pub async fn get_tile_content( - sources: &[&dyn Source], - info: TileInfo, - xyz: TileCoord, - query: Option<&str>, - encodings: Option<&AcceptEncoding>, -) -> ActixResult { - if sources.is_empty() { - return Err(ErrorNotFound("No valid sources found")); + Ok(Self { + sources, + info, + query_str, + query_obj, + encodings, + cache, + }) } - let query_str = query.filter(|v| !v.is_empty()); - let query = match query_str { - Some(v) => Some(Query::::from_query(v)?.into_inner()), - None => None, - }; - let mut tiles = try_join_all(sources.iter().map(|s| s.get_tile(xyz, query.as_ref()))) - .await - .map_err(map_internal_error)?; + pub async fn get_http_response(&self, xyz: TileCoord) -> ActixResult { + let tile = self.get_tile_content(xyz).await?; - let mut layer_count = 0; - let mut last_non_empty_layer = 0; - for (idx, tile) in tiles.iter().enumerate() { - if !tile.is_empty() { - layer_count += 1; - last_non_empty_layer = idx; - } + Ok(if tile.data.is_empty() { + HttpResponse::NoContent().finish() + } else { + let mut response = HttpResponse::Ok(); + response.content_type(tile.info.format.content_type()); + if let Some(val) = tile.info.encoding.content_encoding() { + response.insert_header((CONTENT_ENCODING, val)); + } + response.body(tile.data) + }) } - // Minor optimization to prevent concatenation if there are less than 2 tiles - let data = match layer_count { - 1 => tiles.swap_remove(last_non_empty_layer), - 0 => return Ok(Tile::new(Vec::new(), info)), - _ => { - // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level - // TODO: can zlib, brotli, or zstd be concatenated? - // TODO: implement decompression step for other concatenate-able formats - let can_join = info.format == Format::Mvt - && (info.encoding == Encoding::Uncompressed || info.encoding == Encoding::Gzip); - if !can_join { - return Err(ErrorBadRequest(format!( - "Can't merge {info} tiles. Make sure there is only one non-empty tile source at zoom level {}", - xyz.z - )))?; + pub async fn get_tile_content(&self, xyz: TileCoord) -> ActixResult { + let mut tiles = try_join_all(self.sources.iter().map(|s| async { + get_or_insert_cached_value!( + self.cache, + CacheValue::Tile, + s.get_tile(xyz, self.query_obj.as_ref()), + { + let id = s.get_id().to_owned(); + if let Some(query_str) = self.query_str { + CacheKey::TileWithQuery(id, xyz, query_str.to_owned()) + } else { + CacheKey::Tile(id, xyz) + } + } + ) + })) + .await + .map_err(map_internal_error)?; + + let mut layer_count = 0; + let mut last_non_empty_layer = 0; + for (idx, tile) in tiles.iter().enumerate() { + if !tile.is_empty() { + layer_count += 1; + last_non_empty_layer = idx; } - tiles.concat() } - }; - // decide if (re-)encoding of the tile data is needed, and recompress if so - let tile = recompress(Tile::new(data, info), encodings)?; + // Minor optimization to prevent concatenation if there are less than 2 tiles + let data = match layer_count { + 1 => tiles.swap_remove(last_non_empty_layer), + 0 => return Ok(Tile::new(Vec::new(), self.info)), + _ => { + // Make sure tiles can be concatenated, or if not, that there is only one non-empty tile for each zoom level + // TODO: can zlib, brotli, or zstd be concatenated? + // TODO: implement decompression step for other concatenate-able formats + let can_join = self.info.format == Format::Mvt + && (self.info.encoding == Encoding::Uncompressed + || self.info.encoding == Encoding::Gzip); + if !can_join { + return Err(ErrorBadRequest(format!( + "Can't merge {} tiles. Make sure there is only one non-empty tile source at zoom level {}", + self.info, + xyz.z + )))?; + } + tiles.concat() + } + }; - Ok(tile) -} + // decide if (re-)encoding of the tile data is needed, and recompress if so + self.recompress(data) + } -fn recompress(mut tile: Tile, accept_enc: Option<&AcceptEncoding>) -> ActixResult { - if let Some(accept_enc) = accept_enc { - if tile.info.encoding.is_encoded() { - // already compressed, see if we can send it as is, or need to re-compress - if !accept_enc.iter().any(|e| { - if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { - to_encoding(enc) == Some(tile.info.encoding) - } else { - false + fn recompress(&self, tile: TileData) -> ActixResult { + let mut tile = Tile::new(tile, self.info); + if let Some(accept_enc) = &self.encodings { + if self.info.encoding.is_encoded() { + // already compressed, see if we can send it as is, or need to re-compress + if !accept_enc.iter().any(|e| { + if let Preference::Specific(HeaderEnc::Known(enc)) = e.item { + to_encoding(enc) == Some(tile.info.encoding) + } else { + false + } + }) { + // need to re-compress the tile - uncompress it first + tile = decode(tile)?; } - }) { - // need to re-compress the tile - uncompress it first - tile = decode(tile)?; } - } - if tile.info.encoding == Encoding::Uncompressed { - // only apply compression if the content supports it - if let Some(HeaderEnc::Known(enc)) = accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) { - // (re-)compress the tile into the preferred encoding - tile = encode(tile, enc)?; + if tile.info.encoding == Encoding::Uncompressed { + // only apply compression if the content supports it + if let Some(HeaderEnc::Known(enc)) = + accept_enc.negotiate(SUPPORTED_ENCODINGS.iter()) + { + // (re-)compress the tile into the preferred encoding + tile = encode(tile, enc)?; + } } + Ok(tile) + } else { + // no accepted-encoding header, decode the tile if compressed + decode(tile) } - Ok(tile) - } else { - // no accepted-encoding header, decode the tile if compressed - decode(tile) } } @@ -189,7 +234,7 @@ fn decode(tile: Tile) -> ActixResult { }) } -fn to_encoding(val: ContentEncoding) -> Option { +pub fn to_encoding(val: ContentEncoding) -> Option { Some(match val { ContentEncoding::Identity => Encoding::Uncompressed, ContentEncoding::Gzip => Encoding::Gzip, @@ -233,15 +278,9 @@ mod tests { ("empty,non-empty", vec![1_u8, 2, 3]), ("empty,non-empty,empty", vec![1_u8, 2, 3]), ] { - let (src, _, info) = sources.get_sources(source_id, None).unwrap(); + let src = DynTileSource::new(&sources, source_id, None, "", None, None).unwrap(); let xyz = TileCoord { z: 0, x: 0, y: 0 }; - assert_eq!( - expected, - &get_tile_content(src.as_slice(), info, xyz, None, None) - .await - .unwrap() - .data - ); + assert_eq!(expected, &src.get_tile_content(xyz).await.unwrap().data); } } } diff --git a/martin/src/srv/tiles_info.rs b/martin/src/srv/tiles_info.rs old mode 100644 new mode 100755 diff --git a/martin/src/utils/cache.rs b/martin/src/utils/cache.rs new file mode 100755 index 000000000..ff4421569 --- /dev/null +++ b/martin/src/utils/cache.rs @@ -0,0 +1,91 @@ +use moka::future::Cache; +use pmtiles::Directory; + +use crate::{TileCoord, TileData}; + +pub type MainCache = Cache; +pub type OptMainCache = Option; +pub const NO_MAIN_CACHE: OptMainCache = None; + +#[derive(Debug, Hash, PartialEq, Eq)] +pub enum CacheKey { + /// (pmtiles_id, offset) + PmtDirectory(usize, usize), + /// (source_id, xyz) + Tile(String, TileCoord), + /// (source_id, xyz, url_query) + TileWithQuery(String, TileCoord, String), +} + +#[derive(Debug, Clone)] +pub enum CacheValue { + Tile(TileData), + PmtDirectory(Directory), +} + +macro_rules! trace_cache { + ($typ: literal, $cache: expr, $key: expr) => { + trace!( + "Cache {} for {:?} in {:?} that has {} entries taking up {} space", + $typ, + $key, + $cache.name(), + $cache.entry_count(), + $cache.weighted_size(), + ); + }; +} + +macro_rules! from_cache_value { + ($value_type: path, $data: expr, $key: expr) => { + if let $value_type(data) = $data { + data + } else { + panic!("Unexpected value type {:?} for key {:?} cache", $data, $key) + } + }; +} +#[cfg(feature = "pmtiles")] +macro_rules! get_cached_value { + ($cache: expr, $value_type: path, $make_key: expr) => { + if let Some(cache) = $cache { + let key = $make_key; + if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + Some($crate::utils::cache::from_cache_value!( + $value_type, + data, + key + )) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + None + } + } else { + None + } + }; +} + +macro_rules! get_or_insert_cached_value { + ($cache: expr, $value_type: path, $make_item:expr, $make_key: expr) => {{ + if let Some(cache) = $cache { + let key = $make_key; + Ok(if let Some(data) = cache.get(&key).await { + $crate::utils::cache::trace_cache!("HIT", cache, key); + $crate::utils::cache::from_cache_value!($value_type, data, key) + } else { + $crate::utils::cache::trace_cache!("MISS", cache, key); + let data = $make_item.await?; + cache.insert(key, $value_type(data.clone())).await; + data + }) + } else { + $make_item.await + } + }}; +} + +#[cfg(feature = "pmtiles")] +pub(crate) use get_cached_value; +pub(crate) use {from_cache_value, get_or_insert_cached_value, trace_cache}; diff --git a/martin/src/utils/mod.rs b/martin/src/utils/mod.rs index e0444cdee..306da7e55 100644 --- a/martin/src/utils/mod.rs +++ b/martin/src/utils/mod.rs @@ -1,3 +1,6 @@ +pub(crate) mod cache; +pub use cache::{CacheKey, CacheValue, MainCache, OptMainCache, NO_MAIN_CACHE}; + mod cfg_containers; pub use cfg_containers::{OptBoolObj, OptOneMany}; diff --git a/martin/src/utils/xyz.rs b/martin/src/utils/xyz.rs index 421ec6df6..1a968207e 100644 --- a/martin/src/utils/xyz.rs +++ b/martin/src/utils/xyz.rs @@ -1,6 +1,6 @@ use std::fmt::{Display, Formatter}; -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct TileCoord { pub z: u8, pub x: u32, diff --git a/martin/tests/mb_server_test.rs b/martin/tests/mb_server_test.rs index b20b64f5b..b38d490b6 100644 --- a/martin/tests/mb_server_test.rs +++ b/martin/tests/mb_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pg_server_test.rs b/martin/tests/pg_server_test.rs index ebbc5114a..55b7f37a3 100644 --- a/martin/tests/pg_server_test.rs +++ b/martin/tests/pg_server_test.rs @@ -26,6 +26,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) @@ -1086,6 +1087,7 @@ tables: .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/pmt_server_test.rs b/martin/tests/pmt_server_test.rs index 5b1a4d197..9a6e7ce46 100644 --- a/martin/tests/pmt_server_test.rs +++ b/martin/tests/pmt_server_test.rs @@ -22,6 +22,7 @@ macro_rules! create_app { .app_data(actix_web::web::Data::new( ::martin::srv::Catalog::new(&state).unwrap(), )) + .app_data(actix_web::web::Data::new(::martin::NO_MAIN_CACHE)) .app_data(actix_web::web::Data::new(state.tiles)) .configure(::martin::srv::router), ) diff --git a/martin/tests/utils/pg_utils.rs b/martin/tests/utils/pg_utils.rs index 334b1179f..02116664b 100644 --- a/martin/tests/utils/pg_utils.rs +++ b/martin/tests/utils/pg_utils.rs @@ -1,6 +1,6 @@ use indoc::formatdoc; pub use martin::args::Env; -use martin::{Config, IdResolver, ServerState, Source}; +use martin::{Config, ServerState, Source}; use crate::mock_cfg; @@ -22,7 +22,7 @@ pub fn mock_pgcfg(yaml: &str) -> Config { #[allow(dead_code)] pub async fn mock_sources(mut config: Config) -> MockSource { - let res = config.resolve(IdResolver::default()).await; + let res = config.resolve().await; let res = res.unwrap_or_else(|e| panic!("Failed to resolve config {config:?}: {e}")); (res, config) } diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index f30213609..1f22df562 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -2,7 +2,7 @@ lints.workspace = true [package] name = "mbtiles" -version = "0.9.0" +version = "0.9.1" authors = ["Yuri Astrakhan ", "MapLibre contributors"] description = "A simple low-level MbTiles access and processing library, with some tile format detection and other relevant heuristics." keywords = ["mbtiles", "maps", "tiles", "mvt", "tilejson"] diff --git a/tests/config.yaml b/tests/config.yaml index 7a3f28488..70b08a05c 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -8,6 +8,9 @@ listen_addresses: '0.0.0.0:3000' # Number of web server workers worker_processes: 8 +# Amount of memory (in MB) to use for caching tiles [default: 512, 0 to disable] +cache_size_mb: 8 + # Database configuration. This can also be a list of PG configs. postgres: # Database connection string @@ -166,7 +169,6 @@ postgres: pmtiles: - dir_cache_size_mb: 100 paths: - http://localhost:5412/webp2.pmtiles sources: diff --git a/tests/expected/configured/save_config.yaml b/tests/expected/configured/save_config.yaml index 47379599a..308490c10 100644 --- a/tests/expected/configured/save_config.yaml +++ b/tests/expected/configured/save_config.yaml @@ -1,3 +1,4 @@ +cache_size_mb: 8 keep_alive: 75 listen_addresses: localhost:3111 worker_processes: 1 @@ -165,7 +166,6 @@ pmtiles: pmt: tests/fixtures/pmtiles/stamen_toner__raster_CC-BY+ODbL_z3.pmtiles pmt2: http://localhost:5412/webp2.pmtiles webp2: http://localhost:5412/webp2.pmtiles - dir_cache_size_mb: 100 sprites: paths: tests/fixtures/sprites/src1 sources: From 1cf2a6a6fece39999fe795830605fbb963c34bbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 02:32:38 +0000 Subject: [PATCH 04/19] chore(deps): Bump serde_json from 1.0.108 to 1.0.109 (#1113) Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.108 to 1.0.109.
Release notes

Sourced from serde_json's releases.

v1.0.109

  • Documentation improvements
Commits
  • f88bf1f Release 1.0.109
  • bb62c73 Merge pull request #1097 from serde-rs/doccfg
  • df36d10 Restore doc cfg on re-exports
  • c367091 Merge pull request #1095 from dtolnay/hashtest
  • b328ee7 Eliminate hash closure in favor of calling hash_one directly
  • b9bcbad Use BuildHasher::hash_one
  • 7ff6c9e Use random hasher state for number hashing test
  • fe031cd Delete trace_macros! functionality from test
  • 05196ca Update ui test suite to nightly-2023-11-19
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=serde_json&package-manager=cargo&previous-version=1.0.108&new-version=1.0.109)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e789b0ffe..ef2a60c0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3341,9 +3341,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" dependencies = [ "itoa", "ryu", From edc6e6b969a19fad301a1ca9006109d52657ee5a Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 2 Jan 2024 13:10:45 -0500 Subject: [PATCH 05/19] lock --- Cargo.lock | 92 +++++++++++++++++++++++++++--------------------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef2a60c0f..9c7cf70ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -194,7 +194,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca87830a3e3fb156dc96cfbd31cb620265dd053be734723f22b760d6cc3c3051" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "approx" @@ -384,18 +384,18 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] name = "async-trait" -version = "0.1.76" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -691,7 +691,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -926,7 +926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e" dependencies = [ "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -950,7 +950,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -961,7 +961,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -1123,7 +1123,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -1432,7 +1432,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -2434,7 +2434,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5", "structmeta", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.72" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a293318316cf6478ec1ad2a21c49390a8d5b5eae9fab736467d93fbc0edc29c5" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" dependencies = [ "unicode-ident", ] @@ -2849,9 +2849,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -3126,7 +3126,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.43", + "syn 2.0.46", "unicode-ident", ] @@ -3312,38 +3312,38 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] name = "serde_json" -version = "1.0.109" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" dependencies = [ "itoa", "ryu", @@ -3409,14 +3409,14 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] name = "serde_yaml" -version = "0.9.29" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15e0ef66bf939a7c890a0bf6d5a733c70202225f9888a89ed5c62298b019129" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ "indexmap 2.1.0", "itoa", @@ -3860,7 +3860,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -3871,7 +3871,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -3938,9 +3938,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.43" +version = "2.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" dependencies = [ "proc-macro2", "quote", @@ -4004,22 +4004,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.53" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2cd5904763bad08ad5513ddbb12cf2ae273ca53fa9f68e843e236ec6dfccc09" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.53" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcf4a824cce0aeacd6f38ae6f24234c8e80d68632338ebaa1443b5df9e29e19" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -4141,7 +4141,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -4254,7 +4254,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] @@ -4558,7 +4558,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", "wasm-bindgen-shared", ] @@ -4592,7 +4592,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4942,7 +4942,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.46", ] [[package]] From e3b502d91c24faf42a2bb4c8e90609a76cd5e501 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 02:28:23 +0000 Subject: [PATCH 06/19] chore(deps): Bump serde_json from 1.0.110 to 1.0.111 (#1116) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [serde_json](https://github.com/serde-rs/json) from 1.0.110 to 1.0.111.
Release notes

Sourced from serde_json's releases.

v1.0.111

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=serde_json&package-manager=cargo&previous-version=1.0.110&new-version=1.0.111)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c7cf70ef..8559a9152 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3341,9 +3341,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", From f45ce0368b167318c4612dd326d16301d57cda4f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 02:29:43 +0000 Subject: [PATCH 07/19] chore(deps): Bump clap from 4.4.12 to 4.4.13 (#1117) Bumps [clap](https://github.com/clap-rs/clap) from 4.4.12 to 4.4.13.
Release notes

Sourced from clap's releases.

v4.4.13

[4.4.13] - 2024-01-04

Documentation

  • Fix link to structopt migration guide
Changelog

Sourced from clap's changelog.

[4.4.13] - 2024-01-04

Documentation

  • Fix link to structopt migration guide
Commits
  • 2ab48b2 chore: Release
  • 7a06a8c docs: Update changelog
  • cca190e docs: Correct link to StructOpt migration guide
  • 5c31f45 Merge pull request #5281 from Manishearth/safety-docs
  • ddae7e6 Correct safety docs
  • 48d28aa chore: Release
  • 748ce18 docs: Update changelog
  • adbe6ec Merge pull request #5278 from henry-hsieh/fix-nosort
  • 2b48858 fix: Skip nosort option below bash 4.4
  • 777b744 Merge pull request #5277 from clap-rs/renovate/actions-setup-python-5.x
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=clap&package-manager=cargo&previous-version=4.4.12&new-version=4.4.13)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8559a9152..08739a634 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.12" +version = "4.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfab8ba68f3668e89f6ff60f5b205cea56aa7b769451a59f34b8682f51c056d" +checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" dependencies = [ "clap_builder", "clap_derive", From 521bbd16ed16b5d11420c3fcaef10f0261349f08 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sun, 7 Jan 2024 18:32:57 -0500 Subject: [PATCH 08/19] lock file --- Cargo.lock | 68 +++++++++++++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08739a634..d6d14742d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,7 +80,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -194,7 +194,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -384,7 +384,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -395,7 +395,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -691,7 +691,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -783,9 +783,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -926,7 +926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e" dependencies = [ "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -950,7 +950,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -961,7 +961,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1123,7 +1123,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1432,7 +1432,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1915,9 +1915,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libdeflate-sys" @@ -2434,7 +2434,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5", "structmeta", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.74" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -3126,7 +3126,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.46", + "syn 2.0.48", "unicode-ident", ] @@ -3321,22 +3321,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -3409,7 +3409,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -3860,7 +3860,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -3871,7 +3871,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -3938,9 +3938,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.46" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -4019,7 +4019,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4141,7 +4141,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4254,7 +4254,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4558,7 +4558,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -4592,7 +4592,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4942,7 +4942,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] From c7d8648bfec1df0b0a779a10241ccb8106f0e6ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jan 2024 02:45:28 +0000 Subject: [PATCH 09/19] chore(deps): Bump clap from 4.4.13 to 4.4.14 (#1122) Bumps [clap](https://github.com/clap-rs/clap) from 4.4.13 to 4.4.14.
Release notes

Sourced from clap's releases.

v4.4.14

[4.4.14] - 2024-01-08

Documentation

  • Fix find cookbook entry to allow repeats of flags/options

Features

  • Allow num_args(0) on options which allows making them emulate being a flag for position-tracking flags
Changelog

Sourced from clap's changelog.

[4.4.14] - 2024-01-08

Documentation

  • Fix find cookbook entry to allow repeats of flags/options

Features

  • Allow num_args(0) on options which allows making them emulate being a flag for position-tracking flags
Commits
  • 514f28b chore: Release
  • 1440456 docs: Update changelog
  • d6479ff Merge pull request #5290 from epage/or
  • 02f8214 docs(cookbook): Clarify intent of fake flags
  • c603f34 Merge pull request #5275 from epage/ci
  • 641b42b chore(ci): Speed up critical path of CI
  • 3eaf1af fix(help): Correctly show help for fake flags
  • d63106b docs(cookbook): Allow repeated operators
  • 148e102 fix(builder): Allow custom flag definitions
  • d53d881 docs(cookbook): Demonstrate bad 'find' behavior
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=clap&package-manager=cargo&previous-version=4.4.13&new-version=4.4.14)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6d14742d..bbf17fdd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.13" +version = "4.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bdc885e4cacc7f7c9eedc1ef6da641603180c783c41a15c264944deeaab642" +checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2" dependencies = [ "clap_builder", "clap_derive", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.12" +version = "4.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb7fb5e4e979aec3be7791562fcba452f94ad85e954da024396433e0e25a79e9" +checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370" dependencies = [ "anstream", "anstyle", From 6e5392933085cc496f617177b9274a5aad635124 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Jan 2024 00:59:47 -0500 Subject: [PATCH 10/19] chore(deps): Bump follow-redirects from 1.15.3 to 1.15.4 in /demo/frontend (#1123) Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.3 to 1.15.4.
Commits
  • 6585820 Release version 1.15.4 of the npm package.
  • 7a6567e Disallow bracketed hostnames.
  • 05629af Prefer native URL instead of deprecated url.parse.
  • 1cba8e8 Prefer native URL instead of legacy url.resolve.
  • 72bc2a4 Simplify _processResponse error handling.
  • 3d42aec Add bracket tests.
  • bcbb096 Do not directly set Error properties.
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=follow-redirects&package-manager=npm_and_yarn&previous-version=1.15.3&new-version=1.15.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/maplibre/martin/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- demo/frontend/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demo/frontend/yarn.lock b/demo/frontend/yarn.lock index 35fb2536d..1af4e18a1 100644 --- a/demo/frontend/yarn.lock +++ b/demo/frontend/yarn.lock @@ -2796,9 +2796,9 @@ flatted@^3.2.9: integrity sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ== follow-redirects@^1.15.0: - version "1.15.3" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.3.tgz#fe2f3ef2690afce7e82ed0b44db08165b207123a" - integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q== + version "1.15.4" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf" + integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw== for-each@^0.3.3: version "0.3.3" From a02432a79e3d31e5cdfd5dbb155268192d4007a0 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 9 Jan 2024 14:13:21 -0500 Subject: [PATCH 11/19] update lock --- Cargo.lock | 132 ++++++++++------------------------------------------- 1 file changed, 24 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbf17fdd7..3b81a39f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -440,9 +440,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9" [[package]] name = "base64ct" @@ -714,14 +714,14 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "console" -version = "0.15.7" +version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c926e00cc70edefdc64d3a5ff31cc65bb97a3460097762bd23afb4d8145fccf8" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ "encode_unicode", "lazy_static", "libc", - "windows-sys 0.45.0", + "windows-sys 0.52.0", ] [[package]] @@ -854,54 +854,46 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca89a0e215bab21874660c67903c5f143333cab1da83d041c7ded6053774751" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" dependencies = [ - "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.17" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e3681d554572a651dda4186cd47240627c3d0114d45a95f6ad27f2f22e7548d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crunch" @@ -1492,9 +1484,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -1999,15 +1991,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "mach2" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" -dependencies = [ - "libc", -] - [[package]] name = "martin" version = "0.13.0" @@ -2177,9 +2160,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f353abec74660d4b8533c2516c86eb062f1ec8ca49a2758f4f2b1b60b06b0c6e" +checksum = "2cebde309854872ea4fcaf4d7c870ad8d5873091c6bfb7ce91fd08ea648f20b0" dependencies = [ "async-lock", "async-trait", @@ -2824,13 +2807,12 @@ dependencies = [ [[package]] name = "quanta" -version = "0.11.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" dependencies = [ "crossbeam-utils", "libc", - "mach2", "once_cell", "raw-cpuid", "wasi", @@ -2894,11 +2876,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.7.0" +version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.1", ] [[package]] @@ -4681,15 +4663,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4708,21 +4681,6 @@ dependencies = [ "windows-targets 0.52.0", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -4753,12 +4711,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.0", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4771,12 +4723,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4789,12 +4735,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4807,12 +4747,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4825,12 +4759,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4843,12 +4771,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4861,12 +4783,6 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" From 8c0cc6ae73cd8d3e5fcb190fd47a9d923eaba501 Mon Sep 17 00:00:00 2001 From: Josh Lee Date: Tue, 9 Jan 2024 14:43:48 -0500 Subject: [PATCH 12/19] Make the mbtiles and pmtiles deps optional. (#1124) This also makes the `mbtiles-cp` bin optional. The binary is modestly smaller (by about 6%), but the target directory contains a lot less build products (by about 40%). --------- Co-authored-by: Yuri Astrakhan --- martin/Cargo.toml | 13 +++++++------ martin/src/config.rs | 1 + martin/src/file_config.rs | 1 + martin/src/utils/cache.rs | 6 ++++-- martin/src/utils/error.rs | 2 ++ 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/martin/Cargo.toml b/martin/Cargo.toml index 9e81bec80..24a778ca7 100644 --- a/martin/Cargo.toml +++ b/martin/Cargo.toml @@ -53,6 +53,7 @@ path = "src/bin/martin.rs" [[bin]] name = "martin-cp" path = "src/bin/martin-cp.rs" +required-features = ["mbtiles"] [[bench]] name = "bench" @@ -61,10 +62,10 @@ harness = false [features] default = ["fonts", "mbtiles", "pmtiles", "postgres", "sprites"] fonts = ["dep:bit-set", "dep:pbf_font_tools"] -mbtiles = [] -pmtiles = [] +mbtiles = ["dep:mbtiles"] +pmtiles = ["dep:pmtiles", "dep:reqwest"] postgres = ["dep:deadpool-postgres", "dep:json-patch", "dep:postgis", "dep:postgres", "dep:postgres-protocol", "dep:semver", "dep:tokio-postgres-rustls"] -sprites = ["dep:spreet"] +sprites = ["dep:spreet", "tokio/fs"] bless-tests = [] [dependencies] @@ -84,16 +85,16 @@ itertools.workspace = true json-patch = { workspace = true, optional = true } log.workspace = true martin-tile-utils.workspace = true -mbtiles.workspace = true +mbtiles = { workspace = true, optional = true } moka.workspace = true num_cpus.workspace = true pbf_font_tools = { workspace = true, optional = true } -pmtiles.workspace = true +pmtiles = { workspace = true, optional = true } postgis = { workspace = true, optional = true } postgres-protocol = { workspace = true, optional = true } postgres = { workspace = true, optional = true } regex.workspace = true -reqwest.workspace = true +reqwest = { workspace = true, optional = true } rustls-native-certs.workspace = true rustls-pemfile.workspace = true rustls.workspace = true diff --git a/martin/src/config.rs b/martin/src/config.rs index 8fed9347f..1377c096e 100644 --- a/martin/src/config.rs +++ b/martin/src/config.rs @@ -122,6 +122,7 @@ impl Config { .weigher(|_key, value: &CacheValue| -> u32 { match value { CacheValue::Tile(v) => v.len().try_into().unwrap_or(u32::MAX), + #[cfg(feature = "pmtiles")] CacheValue::PmtDirectory(v) => { v.get_approx_byte_size().try_into().unwrap_or(u32::MAX) } diff --git a/martin/src/file_config.rs b/martin/src/file_config.rs index 3692f75be..d79c008dc 100644 --- a/martin/src/file_config.rs +++ b/martin/src/file_config.rs @@ -43,6 +43,7 @@ pub enum FileError { #[error(r#"Unable to acquire connection to file: {0}"#)] AcquireConnError(String), + #[cfg(feature = "pmtiles")] #[error(r#"PMTiles error {0} processing {1}"#)] PmtError(pmtiles::PmtError, String), } diff --git a/martin/src/utils/cache.rs b/martin/src/utils/cache.rs index ff4421569..d6b5cff24 100755 --- a/martin/src/utils/cache.rs +++ b/martin/src/utils/cache.rs @@ -1,5 +1,4 @@ use moka::future::Cache; -use pmtiles::Directory; use crate::{TileCoord, TileData}; @@ -20,7 +19,8 @@ pub enum CacheKey { #[derive(Debug, Clone)] pub enum CacheValue { Tile(TileData), - PmtDirectory(Directory), + #[cfg(feature = "pmtiles")] + PmtDirectory(pmtiles::Directory), } macro_rules! trace_cache { @@ -38,6 +38,7 @@ macro_rules! trace_cache { macro_rules! from_cache_value { ($value_type: path, $data: expr, $key: expr) => { + #[allow(irrefutable_let_patterns)] if let $value_type(data) = $data { data } else { @@ -45,6 +46,7 @@ macro_rules! from_cache_value { } }; } + #[cfg(feature = "pmtiles")] macro_rules! get_cached_value { ($cache: expr, $value_type: path, $make_key: expr) => { diff --git a/martin/src/utils/error.rs b/martin/src/utils/error.rs index 4d80ebe74..989a078bb 100644 --- a/martin/src/utils/error.rs +++ b/martin/src/utils/error.rs @@ -3,6 +3,7 @@ use std::fmt::Write as _; use std::io; use std::path::PathBuf; +#[cfg(feature = "mbtiles")] use mbtiles::MbtError; use crate::file_config::FileError; @@ -61,6 +62,7 @@ pub enum MartinError { #[error(transparent)] PostgresError(#[from] crate::pg::PgError), + #[cfg(feature = "mbtiles")] #[error(transparent)] MbtilesError(#[from] MbtError), From d742cd59730a82537e3b07e6ace4b5b3f4b0840a Mon Sep 17 00:00:00 2001 From: Lucas Date: Fri, 12 Jan 2024 00:55:28 +0800 Subject: [PATCH 13/19] Add martin-cp to homebrew (#1126) Try to fix [#19](https://github.com/maplibre/homebrew-martin/issues/19) --------- Co-authored-by: Yuri Astrakhan --- .github/files/homebrew.martin.rb.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/files/homebrew.martin.rb.j2 b/.github/files/homebrew.martin.rb.j2 index c8bcbd033..1743c289c 100644 --- a/.github/files/homebrew.martin.rb.j2 +++ b/.github/files/homebrew.martin.rb.j2 @@ -35,6 +35,7 @@ class Martin < Formula def install bin.install "martin" + bin.install "martin-cp" bin.install "mbtiles" end @@ -47,6 +48,7 @@ class Martin < Formula test do `#{bin}/martin --version` + `#{bin}/martin-cp --version` `#{bin}/mbtiles --version` end end From a67febbf5fe9c7cc038001374d62a6de0bded426 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 Jan 2024 02:45:36 +0000 Subject: [PATCH 14/19] chore(deps): Bump clap from 4.4.14 to 4.4.15 (#1130) Bumps [clap](https://github.com/clap-rs/clap) from 4.4.14 to 4.4.15.
Release notes

Sourced from clap's releases.

v4.4.15

[4.4.15] - 2024-01-11

Fixes

  • Improve error for args_conflicts_with_subcommands
  • Ensure we error for args_conflicts_with_subcommands when using subcommand short and long flags
Changelog

Sourced from clap's changelog.

[4.4.15] - 2024-01-11

Fixes

  • Improve error for args_conflicts_with_subcommands
  • Ensure we error for args_conflicts_with_subcommands when using subcommand short and long flags
Commits
  • 12b5c39 chore: Release
  • 1c5b633 docs: Update changelog
  • a5d4641 Merge pull request #5298 from epage/conflict
  • f529ec3 fix(parser): Ensure subcommand flags can conflict
  • a7e04a5 fix(parser): Improve subcommand conflict error
  • ea00ef3 refactor(error): Allow more conflict sources
  • e47d8a2 refactor(parser): Clarify arg error
  • 69c0509 test(parser): Verify conflicts with precedence
  • e2b18f1 test(parser): Show flag behavior
  • 06bff1c test(parser): Check subcommands conflict with positionals
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=clap&package-manager=cargo&previous-version=4.4.14&new-version=4.4.15)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b81a39f4..42e5fab46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.14" +version = "4.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e92c5c1a78c62968ec57dbc2440366a2d6e5a23faf829970ff1585dc6b18e2" +checksum = "c12ed66a79a555082f595f7eb980d08669de95009dd4b3d61168c573ebe38fc9" dependencies = [ "clap_builder", "clap_derive", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.14" +version = "4.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4323769dc8a61e2c39ad7dc26f6f2800524691a44d74fe3d1071a5c24db6370" +checksum = "0f4645eab3431e5a8403a96bea02506a8b35d28cd0f0330977dd5d22f9c84f43" dependencies = [ "anstream", "anstyle", From 4bac7a6f929f1f426fc36d580416a329ef598c90 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 12 Jan 2024 00:22:57 -0500 Subject: [PATCH 15/19] lock --- Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 42e5fab46..91af37b6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -278,9 +278,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" dependencies = [ "anstyle", "anstyle-parse", @@ -440,9 +440,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79fed4cdb43e993fcdadc7e58a09fd0e3e649c4436fa11da71c9f1f3ee7feb9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64ct" @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.15" +version = "4.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c12ed66a79a555082f595f7eb980d08669de95009dd4b3d61168c573ebe38fc9" +checksum = "58e54881c004cec7895b0068a0a954cd5d62da01aef83fa35b1e594497bf5445" dependencies = [ "clap_builder", "clap_derive", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.15" +version = "4.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4645eab3431e5a8403a96bea02506a8b35d28cd0f0330977dd5d22f9c84f43" +checksum = "59cb82d7f531603d2fd1f507441cdd35184fa81beff7bd489570de7f773460bb" dependencies = [ "anstream", "anstyle", @@ -1517,9 +1517,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "b553656127a00601c8ae5590fcfdc118e4083a7924b6cf4ffc1ea4b99dc429d7" dependencies = [ "bytes", "fnv", @@ -3977,9 +3977,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] From eebfc637839b8887af47a157927fc03dc9d4c326 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sun, 14 Jan 2024 20:21:30 -0500 Subject: [PATCH 16/19] add PGPARAMS to just for testing connections (#1131) A small justfile improvement --- justfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/justfile b/justfile index 7bd68cbc8..56a95442d 100644 --- a/justfile +++ b/justfile @@ -4,8 +4,11 @@ set shell := ["bash", "-c"] #export DATABASE_URL="postgres://postgres:postgres@localhost:5411/db" -export PGPORT := "5411" -export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db" +# Set additional database connection parameters, e.g. just PGPARAMS='keepalives=0&keepalives_idle=15' psql +PGPARAMS := "" +PGPORT := "5411" + +export DATABASE_URL := "postgres://postgres:postgres@localhost:" + PGPORT + "/db" + (if PGPARAMS != "" { "?" + PGPARAMS } else { "" }) export CARGO_TERM_COLOR := "always" #export RUST_LOG := "debug" @@ -38,11 +41,11 @@ debug-page *ARGS: start # Run PSQL utility against the test database psql *ARGS: - psql {{ ARGS }} {{ DATABASE_URL }} + psql {{ ARGS }} {{ quote(DATABASE_URL) }} # Run pg_dump utility against the test database pg_dump *ARGS: - pg_dump {{ ARGS }} {{ DATABASE_URL }} + pg_dump {{ ARGS }} {{ quote(DATABASE_URL) }} # Perform cargo clean to delete all build files clean: clean-test stop @@ -268,7 +271,7 @@ git *ARGS: start # Print the connection string for the test database print-conn-str: - @echo {{ DATABASE_URL }} + @echo {{ quote(DATABASE_URL) }} # Run cargo fmt and cargo clippy lint: fmt clippy From 01906b57175dc8b4d14668340d5790fc98d7eb34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 02:30:22 +0000 Subject: [PATCH 17/19] chore(deps): Bump clap from 4.4.16 to 4.4.17 (#1133) Bumps [clap](https://github.com/clap-rs/clap) from 4.4.16 to 4.4.17.
Release notes

Sourced from clap's releases.

v4.4.17

[4.4.17] - 2024-01-15

Fixes

  • Fix panic! when mixing args_conflicts_with_subcommands with ArgGroup (which is implicit with derive) introduced in 4.4.15
Changelog

Sourced from clap's changelog.

[4.4.17] - 2024-01-15

Fixes

  • Fix panic! when mixing args_conflicts_with_subcommands with ArgGroup (which is implicit with derive) introduced in 4.4.15
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=clap&package-manager=cargo&previous-version=4.4.16&new-version=4.4.17)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91af37b6c..38e695301 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.16" +version = "4.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e54881c004cec7895b0068a0a954cd5d62da01aef83fa35b1e594497bf5445" +checksum = "80932e03c33999b9235edb8655bc9df3204adc9887c2f95b50cb1deb9fd54253" dependencies = [ "clap_builder", "clap_derive", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.16" +version = "4.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59cb82d7f531603d2fd1f507441cdd35184fa81beff7bd489570de7f773460bb" +checksum = "d6c0db58c659eef1c73e444d298c27322a1b52f6927d2ad470c0c0f96fa7b8fa" dependencies = [ "anstream", "anstyle", From eacafd07d9f7194e63a890ea6e621676a82b8532 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 16 Jan 2024 18:43:08 -0500 Subject: [PATCH 18/19] bump lock --- Cargo.lock | 82 +++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38e695301..709427ed4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,7 +46,7 @@ dependencies = [ "actix-utils", "ahash", "base64", - "bitflags 2.4.1", + "bitflags 2.4.2", "brotli", "bytes", "bytestring", @@ -278,9 +278,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd2405b3ac1faab2990b74d728624cd9fd115651fcecc7c2d8daf01376275ba" +checksum = "628a8f9bd1e24b4e0db2b4bc2d000b001e7dd032d54afa60a68836aeec5aa54a" dependencies = [ "anstyle", "anstyle-parse", @@ -473,9 +473,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" dependencies = [ "serde", ] @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.17" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80932e03c33999b9235edb8655bc9df3204adc9887c2f95b50cb1deb9fd54253" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.17" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c0db58c659eef1c73e444d298c27322a1b52f6927d2ad470c0c0f96fa7b8fa" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -1856,15 +1856,15 @@ dependencies = [ [[package]] name = "jpeg-decoder" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1954,9 +1954,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "local-channel" @@ -2208,7 +2208,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "libc", ] @@ -2571,9 +2571,9 @@ dependencies = [ [[package]] name = "png" -version = "0.17.10" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +checksum = "1f6c3c3e617595665b8ea2ff95a86066be38fb121ff920a9c0eb282abcd1da5a" dependencies = [ "bitflags 1.3.2", "crc32fast", @@ -2880,7 +2880,7 @@ version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", ] [[package]] @@ -3118,7 +3118,7 @@ version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", @@ -3149,11 +3149,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -3520,9 +3520,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e" [[package]] name = "socket2" @@ -3699,7 +3699,7 @@ checksum = "e37195395df71fd068f6e2082247891bc11e3289624bbc776a0cdfa1ca7f1ea4" dependencies = [ "atoi", "base64", - "bitflags 2.4.1", + "bitflags 2.4.2", "byteorder", "bytes", "crc", @@ -3741,7 +3741,7 @@ checksum = "d6ac0ac3b7ccd10cc96c7ab29791a7dd236bd94021f31eec7ba3d46a74aa1c24" dependencies = [ "atoi", "base64", - "bitflags 2.4.1", + "bitflags 2.4.2", "byteorder", "crc", "dotenvy", @@ -4521,9 +4521,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -4531,9 +4531,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -4546,9 +4546,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -4558,9 +4558,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4568,9 +4568,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -4581,15 +4581,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", From 8aea90e18b23badb0b8f0ce4caeade16a5297b26 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 16 Jan 2024 20:42:11 -0500 Subject: [PATCH 19/19] Upgrade to Rustls 0.22 (#1136) Fix #1036 --- Cargo.lock | 138 +++++++++++++++++++++++++++++++++++++------ Cargo.toml | 8 +-- martin/src/pg/tls.rs | 89 +++++++++++++++++++--------- 3 files changed, 186 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 709427ed4..b70b0dfac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -450,6 +450,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bcder" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c627747a6774aab38beb35990d88309481378558875a41da1a4b2e373c906ef0" +dependencies = [ + "bytes", + "smallvec", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -1692,9 +1702,9 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls", + "rustls 0.21.10", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", ] [[package]] @@ -2027,9 +2037,9 @@ dependencies = [ "pprof", "regex", "reqwest", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.22.2", + "rustls-native-certs 0.7.0", + "rustls-pemfile 2.0.0", "semver", "serde", "serde_json", @@ -2442,6 +2452,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "pem" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13fe415cdf3c8e44518e18a7c95a13431d9bdf6d15367d82b23c377fdd441a" +dependencies = [ + "base64", + "serde", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -2982,15 +3002,15 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.21.10", + "rustls-native-certs 0.6.3", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", "system-configuration", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -3168,10 +3188,24 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring 0.17.7", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" +dependencies = [ + "log", + "ring 0.17.7", + "rustls-pki-types", + "rustls-webpki 0.102.1", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -3179,7 +3213,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.0.0", + "rustls-pki-types", "schannel", "security-framework", ] @@ -3193,6 +3240,22 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -3203,6 +3266,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" +dependencies = [ + "ring 0.17.7", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustybuzz" version = "0.10.0" @@ -4154,16 +4228,17 @@ dependencies = [ [[package]] name = "tokio-postgres-rustls" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f" +checksum = "23ca59f99c85e77d01626fa504cd56bcd8ece31c1c9bc218460e3c526690a09f" dependencies = [ "futures", - "ring 0.16.20", - "rustls", + "ring 0.17.7", + "rustls 0.22.2", "tokio", "tokio-postgres", - "tokio-rustls", + "tokio-rustls 0.25.0", + "x509-certificate", ] [[package]] @@ -4172,7 +4247,18 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.10", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.2", + "rustls-pki-types", "tokio", ] @@ -4814,6 +4900,24 @@ dependencies = [ "tap", ] +[[package]] +name = "x509-certificate" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5d27c90840e84503cf44364de338794d5d5680bdd1da6272d13f80b0769ee0" +dependencies = [ + "bcder", + "bytes", + "chrono", + "der", + "hex", + "pem", + "ring 0.16.20", + "signature", + "spki", + "thiserror", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index fa540a7fa..59fb8ffb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,9 +62,9 @@ pretty_assertions = "1" regex = "1" reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-native-roots"] } rstest = "0.18" -rustls = { version = "0.21", features = ["dangerous_configuration"] } -rustls-native-certs = "0.6" -rustls-pemfile = "1" +rustls = "0.22" +rustls-native-certs = "0.7" +rustls-pemfile = "2" semver = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -79,7 +79,7 @@ thiserror = "1" tile-grid = "0.5" tilejson = "0.4" tokio = { version = "1", features = ["macros"] } -tokio-postgres-rustls = "0.10" +tokio-postgres-rustls = "0.11" url = "2.5" [profile.dev.package] diff --git a/martin/src/pg/tls.rs b/martin/src/pg/tls.rs index bdd8981eb..3b0297a02 100644 --- a/martin/src/pg/tls.rs +++ b/martin/src/pg/tls.rs @@ -1,4 +1,5 @@ use std::fs::File; +use std::io; use std::io::BufReader; use std::path::PathBuf; use std::str::FromStr; @@ -7,9 +8,12 @@ use deadpool_postgres::tokio_postgres::config::SslMode; use deadpool_postgres::tokio_postgres::Config; use log::{info, warn}; use regex::Regex; -use rustls::{Certificate, PrivateKey}; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::{verify_tls12_signature, verify_tls13_signature}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, Error, SignatureScheme}; use rustls_native_certs::load_native_certs; -use rustls_pemfile::Item::RSAKey; +use rustls_pemfile::Item::Pkcs1Key; use tokio_postgres_rustls::MakeRustlsConnect; use crate::pg::PgError::{ @@ -51,28 +55,60 @@ pub fn parse_conn_str(conn_str: &str) -> PgResult<(Config, SslModeOverride)> { Ok((pg_cfg, mode)) } +#[derive(Debug)] struct NoCertificateVerification {} -impl rustls::client::ServerCertVerifier for NoCertificateVerification { +impl ServerCertVerifier for NoCertificateVerification { fn verify_server_cert( &self, - _end_entity: &Certificate, - _intermediates: &[Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, - _ocsp: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls12_signature( + message, + cert, + dss, + &rustls::crypto::ring::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls13_signature( + message, + cert, + dss, + &rustls::crypto::ring::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() } } -fn read_certs(file: &PathBuf) -> PgResult> { - Ok(rustls_pemfile::certs(&mut cert_reader(file)?) - .map_err(|e| CannotParseCert(e, file.clone()))? - .into_iter() - .map(Certificate) - .collect()) +fn read_certs(file: &PathBuf) -> PgResult>> { + rustls_pemfile::certs(&mut cert_reader(file)?) + .collect::, io::Error>>() + .map_err(|e| CannotParseCert(e, file.clone())) } fn cert_reader(file: &PathBuf) -> PgResult> { @@ -108,7 +144,7 @@ pub fn make_connector( if let Some(file) = &pg_certs.ssl_root_cert { for cert in read_certs(file)? { - roots.add(&cert)?; + roots.add(cert)?; } info!("Using {} as a root certificate", file.display()); } @@ -116,22 +152,19 @@ pub fn make_connector( if verify_ca || pg_certs.ssl_root_cert.is_some() || pg_certs.ssl_cert.is_some() { let certs = load_native_certs().map_err(CannotLoadRoots)?; for cert in certs { - roots.add(&Certificate(cert.0))?; + roots.add(cert)?; } } - let builder = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_root_certificates(roots); + let builder = rustls::ClientConfig::builder().with_root_certificates(roots); let mut builder = if let (Some(cert), Some(key)) = (&pg_certs.ssl_cert, &pg_certs.ssl_key) { - match rustls_pemfile::read_one(&mut cert_reader(key)?) - .map_err(|e| CannotParseCert(e, key.clone()))? - { - Some(RSAKey(rsa_key)) => builder - .with_client_auth_cert(read_certs(cert)?, PrivateKey(rsa_key)) + match rustls_pemfile::read_one(&mut cert_reader(key)?) { + Ok(Some(Pkcs1Key(rsa_key))) => builder + .with_client_auth_cert(read_certs(cert)?, rsa_key.into()) .map_err(|e| CannotUseClientKey(e, cert.clone(), key.clone()))?, - _ => Err(InvalidPrivateKey(key.clone()))?, + Ok(_) => Err(InvalidPrivateKey(key.clone()))?, + Err(e) => Err(CannotParseCert(e, key.clone()))?, } } else { if pg_certs.ssl_key.is_some() || pg_certs.ssl_key.is_some() {