diff --git a/Cargo.lock b/Cargo.lock index 0265dfa06a5a..0dcd2c021005 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1666,20 +1666,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" -[[package]] -name = "combine" -version = "4.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" -dependencies = [ - "bytes", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "tokio-util", -] - [[package]] name = "comfy-table" version = "6.1.4" @@ -1770,7 +1756,7 @@ dependencies = [ "goldenfile", "itertools 0.10.5", "logos", - "minitrace 0.6.2", + "minitrace", "nom", "nom-rule", "ordered-float 3.7.0", @@ -1812,7 +1798,7 @@ dependencies = [ "libc", "log", "logcall", - "minitrace 0.6.2", + "minitrace", "num_cpus", "once_cell", "parking_lot 0.12.1", @@ -2216,7 +2202,7 @@ dependencies = [ "common-proto-conv", "common-protos", "common-storage", - "minitrace 0.6.2", + "minitrace", "mockall", "serde_json", ] @@ -2240,7 +2226,7 @@ dependencies = [ "log", "logcall", "maplit", - "minitrace 0.6.2", + "minitrace", "serde", "serde_json", "thiserror", @@ -2298,7 +2284,7 @@ dependencies = [ "lazy_static", "log", "logcall", - "minitrace 0.6.2", + "minitrace", "once_cell", "parking_lot 0.12.1", "prost 0.12.1", @@ -2323,7 +2309,7 @@ dependencies = [ "common-meta-stoerr", "common-meta-types", "log", - "minitrace 0.6.2", + "minitrace", "once_cell", "tempfile", ] @@ -2337,7 +2323,7 @@ dependencies = [ "common-meta-types", "futures-util", "log", - "minitrace 0.6.2", + "minitrace", "serde", "thiserror", "tokio", @@ -2387,7 +2373,7 @@ dependencies = [ "lazy_static", "log", "maplit", - "minitrace 0.6.2", + "minitrace", "num", "once_cell", "openraft 0.8.4", @@ -2414,7 +2400,7 @@ dependencies = [ "common-meta-types", "common-tracing", "log", - "minitrace 0.6.2", + "minitrace", "once_cell", "openraft 0.8.4", "pretty_assertions", @@ -2513,7 +2499,7 @@ dependencies = [ "common-exception", "common-expression", "futures", - "minitrace 0.6.2", + "minitrace", "petgraph", "serde", "serde_json", @@ -2561,7 +2547,7 @@ dependencies = [ "futures", "futures-util", "log", - "minitrace 0.6.2", + "minitrace", "opendal", "parking_lot 0.12.1", "serde", @@ -2606,7 +2592,7 @@ dependencies = [ "convert_case 0.6.0", "enumflags2", "maplit", - "minitrace 0.6.2", + "minitrace", "num", "pretty_assertions", "thiserror", @@ -2716,7 +2702,7 @@ dependencies = [ "indexmap 2.0.0", "itertools 0.10.5", "log", - "minitrace 0.6.2", + "minitrace", "num-derive", "num-traits", "once_cell", @@ -2816,7 +2802,7 @@ dependencies = [ "jsonb 0.3.0 (git+https://github.com/datafuselabs/jsonb?rev=1d7a3e9)", "log", "metrics", - "minitrace 0.6.2", + "minitrace", "opendal", "parquet-format-safe", "rand 0.8.5", @@ -2860,7 +2846,7 @@ dependencies = [ "futures", "hive_metastore", "log", - "minitrace 0.6.2", + "minitrace", "opendal", "ordered-float 3.7.0", "serde", @@ -2894,7 +2880,7 @@ dependencies = [ "futures", "icelake", "match-template", - "minitrace 0.6.2", + "minitrace", "opendal", "parquet", "serde", @@ -3146,7 +3132,7 @@ dependencies = [ "fern", "humantime", "log", - "minitrace 0.6.2", + "minitrace", "minitrace-opentelemetry", "opentelemetry", "opentelemetry-otlp", @@ -3349,12 +3335,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc16" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" - [[package]] name = "crc32fast" version = "1.3.2" @@ -3772,7 +3752,7 @@ dependencies = [ "futures", "limits-rs", "log", - "minitrace 0.6.2", + "minitrace", "opendal", "poem", "sentry", @@ -3870,7 +3850,7 @@ dependencies = [ "log", "logcall", "maplit", - "minitrace 0.6.2", + "minitrace", "once_cell", "poem", "pretty_assertions", @@ -3987,7 +3967,7 @@ dependencies = [ "maplit", "match-template", "metrics", - "minitrace 0.6.2", + "minitrace", "mysql_async", "naive-cityhash", "num", @@ -7932,21 +7912,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "minitrace" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823c2c0be9db5d4a0497700a405a231c9db19e67a24f5fe1ef4e3fdaca6a9b7e" -dependencies = [ - "futures", - "minitrace-macro 0.5.1", - "minstant", - "once_cell", - "parking_lot 0.12.1", - "pin-project", - "rand 0.8.5", -] - [[package]] name = "minitrace" version = "0.6.2" @@ -7954,7 +7919,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9c20b20fba7e39214dbe7c5766d2d74da2fa795d4a1c66487df20a0d9657c67" dependencies = [ "futures", - "minitrace-macro 0.6.2", + "minitrace-macro", "minstant", "once_cell", "parking_lot 0.12.1", @@ -7963,18 +7928,6 @@ dependencies = [ "rtrb", ] -[[package]] -name = "minitrace-macro" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3a906b1d9cacfe2174ad7c4139fccb441af566940559822103b79e5ea25f31" -dependencies = [ - "proc-macro-error 1.0.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "minitrace-macro" version = "0.6.2" @@ -7995,7 +7948,7 @@ checksum = "6a7ebd135c6b6e20a4594c7965f592c087de2705384628c9e986379568789495" dependencies = [ "futures", "log", - "minitrace 0.6.2", + "minitrace", "opentelemetry", ] @@ -8480,9 +8433,9 @@ dependencies = [ [[package]] name = "opendal" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e31b48f0af6de5b3b344c1acc1e06c4581dca3e13cd5ba05269927fc2abf953a" +checksum = "ec1f653f28f2ced5ab8214ebb456674a234ee3379f3e493ab46dd7964eca43f9" dependencies = [ "anyhow", "async-compat", @@ -8495,10 +8448,9 @@ dependencies = [ "futures", "hdrs", "http", - "hyper", "log", "md-5", - "minitrace 0.5.1", + "minitrace", "moka", "once_cell", "parking_lot 0.12.1", @@ -8508,7 +8460,6 @@ dependencies = [ "prometheus-client", "prost 0.11.9", "quick-xml 0.30.0", - "redis", "reqsign", "reqwest", "serde", @@ -9974,33 +9925,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "redis" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff5d95dd18a4d76650f0c2607ed8ebdbf63baf9cb934e1c233cd220c694db1d7" -dependencies = [ - "arc-swap", - "async-trait", - "bytes", - "combine", - "crc16", - "futures", - "futures-util", - "itoa", - "log", - "percent-encoding", - "pin-project-lite", - "rand 0.8.5", - "ryu", - "sha1_smol", - "socket2 0.4.9", - "tokio", - "tokio-retry", - "tokio-util", - "url", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -11350,7 +11274,7 @@ dependencies = [ "common-functions", "criterion", "match-template", - "minitrace 0.6.2", + "minitrace", "rand 0.8.5", "serde", "storages-common-table-meta", diff --git a/Cargo.toml b/Cargo.toml index b0a5ac177b1c..208bbe56c27a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,12 +108,11 @@ members = [ [workspace.dependencies] # databend maintains: sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1", default-features = false } -opendal = { version = "0.41", features = [ +opendal = { version = "0.42", features = [ "layers-minitrace", "layers-prometheus-client", "services-ipfs", "services-moka", - "services-redis", "trust-dns", ] } ethnum = { version = "1.3.2" } @@ -251,7 +250,6 @@ rpath = false [patch.crates-io] # If there are dependencies that need patching, they can be listed below. - arrow-format = { git = "https://github.com/everpcpc/arrow-format", rev = "588d371" } parquet2 = { git = "https://github.com/jorgecarleitao/parquet2", rev = "b0e6545" } metrics = { git = "https://github.com/datafuse-extras/metrics.git", rev = "fc2ecd1" } diff --git a/src/common/storage/src/config.rs b/src/common/storage/src/config.rs index 7ae418172baa..c2910e51ed02 100644 --- a/src/common/storage/src/config.rs +++ b/src/common/storage/src/config.rs @@ -36,9 +36,6 @@ use serde::Serialize; /// [storage.data] /// type = "s3" /// -/// [storage.cache] -/// type = "redis" -/// /// [storage.temporary] /// type = "s3" /// ``` diff --git a/src/common/storage/src/operator.rs b/src/common/storage/src/operator.rs index 208bea149757..946a4cd97f3d 100644 --- a/src/common/storage/src/operator.rs +++ b/src/common/storage/src/operator.rs @@ -36,7 +36,6 @@ use common_meta_app::storage::StorageMokaConfig; use common_meta_app::storage::StorageObsConfig; use common_meta_app::storage::StorageOssConfig; use common_meta_app::storage::StorageParams; -use common_meta_app::storage::StorageRedisConfig; use common_meta_app::storage::StorageS3Config; use common_meta_app::storage::StorageWebhdfsConfig; use log::warn; @@ -72,7 +71,6 @@ pub fn init_operator(cfg: &StorageParams) -> Result { StorageParams::Obs(cfg) => build_operator(init_obs_operator(cfg)?)?, StorageParams::S3(cfg) => build_operator(init_s3_operator(cfg)?)?, StorageParams::Oss(cfg) => build_operator(init_oss_operator(cfg)?)?, - StorageParams::Redis(cfg) => build_operator(init_redis_operator(cfg)?)?, StorageParams::Webhdfs(cfg) => build_operator(init_webhdfs_operator(cfg)?)?, StorageParams::Cos(cfg) => build_operator(init_cos_operator(cfg)?)?, v => { @@ -338,26 +336,6 @@ fn init_moka_operator(v: &StorageMokaConfig) -> Result { Ok(builder) } -/// init_redis_operator will init a reids operator. -fn init_redis_operator(v: &StorageRedisConfig) -> Result { - let mut builder = services::Redis::default(); - - builder.endpoint(&v.endpoint_url); - builder.root(&v.root); - builder.db(v.db); - if let Some(v) = v.default_ttl { - builder.default_ttl(Duration::from_secs(v as u64)); - } - if let Some(v) = &v.username { - builder.username(v); - } - if let Some(v) = &v.password { - builder.password(v); - } - - Ok(builder) -} - /// init_webhdfs_operator will init a WebHDFS operator fn init_webhdfs_operator(v: &StorageWebhdfsConfig) -> Result { let mut builder = services::Webhdfs::default(); diff --git a/src/meta/app/src/storage/storage_params.rs b/src/meta/app/src/storage/storage_params.rs index 61c0366fcc7d..ff7041125ffd 100644 --- a/src/meta/app/src/storage/storage_params.rs +++ b/src/meta/app/src/storage/storage_params.rs @@ -38,7 +38,6 @@ pub enum StorageParams { Obs(StorageObsConfig), Oss(StorageOssConfig), S3(StorageS3Config), - Redis(StorageRedisConfig), Webhdfs(StorageWebhdfsConfig), Cos(StorageCosConfig), @@ -72,7 +71,6 @@ impl StorageParams { StorageParams::Oss(v) => v.endpoint_url.starts_with("https://"), StorageParams::S3(v) => v.endpoint_url.starts_with("https://"), StorageParams::Gcs(v) => v.endpoint_url.starts_with("https://"), - StorageParams::Redis(_) => false, StorageParams::Webhdfs(v) => v.endpoint_url.starts_with("https://"), StorageParams::Cos(v) => v.endpoint_url.starts_with("https://"), StorageParams::None => false, @@ -94,7 +92,6 @@ impl StorageParams { StorageParams::Oss(v) => v.root = f(&v.root), StorageParams::S3(v) => v.root = f(&v.root), StorageParams::Gcs(v) => v.root = f(&v.root), - StorageParams::Redis(v) => v.root = f(&v.root), StorageParams::Webhdfs(v) => v.root = f(&v.root), StorageParams::Cos(v) => v.root = f(&v.root), StorageParams::None => {} @@ -214,13 +211,6 @@ impl Display for StorageParams { v.bucket, v.root, v.endpoint_url ) } - StorageParams::Redis(v) => { - write!( - f, - "redis | db={},root={},endpoint={}", - v.db, v.root, v.endpoint_url - ) - } StorageParams::Webhdfs(v) => { write!(f, "webhdfs | root={},endpoint={}", v.root, v.endpoint_url) } @@ -528,38 +518,6 @@ impl Default for StorageMokaConfig { } } -/// config for Redis Storage Service -#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct StorageRedisConfig { - pub endpoint_url: String, - pub username: Option, - pub password: Option, - pub root: String, - pub db: i64, - /// TTL in seconds - pub default_ttl: Option, -} - -impl Debug for StorageRedisConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut d = f.debug_struct("StorageRedisConfig"); - - d.field("endpoint_url", &self.endpoint_url) - .field("db", &self.db) - .field("root", &self.root) - .field("default_ttl", &self.default_ttl); - - if let Some(username) = &self.username { - d.field("username", &mask_string(username, 3)); - } - if let Some(password) = &self.password { - d.field("password", &mask_string(password, 3)); - } - - d.finish() - } -} - /// config for WebHDFS Storage Service #[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct StorageWebhdfsConfig { diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index 4a9dbe1e42d3..e04a4d25dff1 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -38,7 +38,6 @@ use common_meta_app::storage::StorageMokaConfig as InnerStorageMokaConfig; use common_meta_app::storage::StorageObsConfig as InnerStorageObsConfig; use common_meta_app::storage::StorageOssConfig as InnerStorageOssConfig; use common_meta_app::storage::StorageParams; -use common_meta_app::storage::StorageRedisConfig as InnerStorageRedisConfig; use common_meta_app::storage::StorageS3Config as InnerStorageS3Config; use common_meta_app::storage::StorageWebhdfsConfig as InnerStorageWebhdfsConfig; use common_meta_app::tenant::TenantQuota; @@ -238,9 +237,6 @@ impl Config { /// [storage.data] /// type = "s3" /// -/// [storage.cache] -/// type = "redis" -/// /// [storage.temporary] /// type = "s3" /// ``` @@ -1171,64 +1167,6 @@ impl TryInto for MokaStorageConfig { } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] -#[serde(default)] -pub struct RedisStorageConfig { - pub endpoint_url: String, - pub username: String, - pub password: String, - pub root: String, - pub db: i64, - /// TTL in seconds - pub default_ttl: i64, -} - -impl Default for RedisStorageConfig { - fn default() -> Self { - InnerStorageRedisConfig::default().into() - } -} - -impl From for RedisStorageConfig { - fn from(v: InnerStorageRedisConfig) -> Self { - Self { - endpoint_url: v.endpoint_url.clone(), - username: v.username.unwrap_or_default(), - password: v.password.unwrap_or_default(), - root: v.root.clone(), - db: v.db, - default_ttl: v.default_ttl.unwrap_or_default(), - } - } -} - -impl TryInto for RedisStorageConfig { - type Error = ErrorCode; - - fn try_into(self) -> Result { - Ok(InnerStorageRedisConfig { - endpoint_url: self.endpoint_url.clone(), - username: if self.username.is_empty() { - None - } else { - Some(self.username.clone()) - }, - password: if self.password.is_empty() { - None - } else { - Some(self.password.clone()) - }, - root: self.root.clone(), - db: self.db, - default_ttl: if self.default_ttl == 0 { - None - } else { - Some(self.default_ttl) - }, - }) - } -} - #[derive(Clone, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct WebhdfsStorageConfig { diff --git a/src/query/sharing/src/layer.rs b/src/query/sharing/src/layer.rs index 0985393be5fd..b3ad711542f3 100644 --- a/src/query/sharing/src/layer.rs +++ b/src/query/sharing/src/layer.rs @@ -171,7 +171,10 @@ impl Accessor for SharedAccessor { let content_length = parse_content_length(resp.headers()) .unwrap() .expect("content_length must be valid"); - Ok((RpRead::new(content_length), resp.into_body())) + Ok(( + RpRead::new().with_size(Some(content_length)), + resp.into_body(), + )) } else { Err(parse_error(resp).await) }