From 14c83db6bcea753f6b8477ac642b7ae5386b1988 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 31 Dec 2024 10:06:26 -0500 Subject: [PATCH 1/6] Update examples README.md --- datafusion-examples/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index 23cf8830e36d..ae404ecce324 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -53,6 +53,7 @@ cargo run --example dataframe - [`analyzer_rule.rs`](examples/analyzer_rule.rs): Use a custom AnalyzerRule to change a query's semantics (row level access control) - [`catalog.rs`](examples/catalog.rs): Register the table into a custom catalog - [`composed_extension_codec`](examples/composed_extension_codec.rs): Example of using multiple extension codecs for serialization / deserialization +- [`config_extensions`](examples/composed_extension_codec.rs): Pass custom options with `ConfigOptions` - [`csv_sql_streaming.rs`](examples/csv_sql_streaming.rs): Build and run a streaming query plan from a SQL statement against a local CSV file - [`custom_datasource.rs`](examples/custom_datasource.rs): Run queries against a custom datasource (TableProvider) - [`custom_file_format.rs`](examples/custom_file_format.rs): Write data to a custom file format From 30becba287872aaa9fb0e6d98a48409bc89eafc8 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 31 Dec 2024 10:14:14 -0500 Subject: [PATCH 2/6] Minor: consolidate ConfigExtension example into API docs --- .../examples/config_extension.rs | 52 ------------------- datafusion/common/src/config.rs | 46 +++++++++++++++- 2 files changed, 45 insertions(+), 53 deletions(-) delete mode 100644 datafusion-examples/examples/config_extension.rs diff --git a/datafusion-examples/examples/config_extension.rs b/datafusion-examples/examples/config_extension.rs deleted file mode 100644 index b9f83f91ce56..000000000000 --- a/datafusion-examples/examples/config_extension.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! This example demonstrates how to extend the DataFusion configs with custom extensions. - -use datafusion::{ - common::{config::ConfigExtension, extensions_options}, - config::ConfigOptions, -}; - -extensions_options! { - /// My own config options. - pub struct MyConfig { - /// Should "foo" be replaced by "bar"? - pub foo_to_bar: bool, default = true - - /// How many "baz" should be created? - pub baz_count: usize, default = 1337 - } -} - -impl ConfigExtension for MyConfig { - const PREFIX: &'static str = "my_config"; -} - -fn main() { - // set up config struct and register extension - let mut config = ConfigOptions::default(); - config.extensions.insert(MyConfig::default()); - - // overwrite config default - config.set("my_config.baz_count", "42").unwrap(); - - // check config state - let my_config = config.extensions.get::().unwrap(); - assert!(my_config.foo_to_bar,); - assert_eq!(my_config.baz_count, 42,); -} diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 942aa308e200..c7faeb018737 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -889,8 +889,50 @@ impl ConfigOptions { } } -/// [`ConfigExtension`] provides a mechanism to store third-party configuration within DataFusion +/// [`ConfigExtension`] provides a mechanism to store third-party configuration +/// within DataFusion [`ConfigOptions`] /// +/// This mechanism can be used to pass configuration to user defined functions +/// or optimizer passes +/// +/// # Example +/// ``` +/// use datafusion_common::{ +/// config::ConfigExtension, extensions_options, +/// config::ConfigOptions, +/// }; +/// // Define a new configuration struct using the `extensions_options` macro +/// extensions_options! { +/// /// My own config options. +/// pub struct MyConfig { +/// /// Should "foo" be replaced by "bar"? +/// pub foo_to_bar: bool, default = true +/// +/// /// How many "baz" should be created? +/// pub baz_count: usize, default = 1337 +/// } +/// } +/// +/// impl ConfigExtension for MyConfig { +/// const PREFIX: &'static str = "my_config"; +/// } +/// +/// fn main() { +/// // set up config struct and register extension +/// let mut config = ConfigOptions::default(); +/// config.extensions.insert(MyConfig::default()); +/// +/// // overwrite config default +/// config.set("my_config.baz_count", "42").unwrap(); +/// +/// // check config state +/// let my_config = config.extensions.get::().unwrap(); +/// assert!(my_config.foo_to_bar,); +/// assert_eq!(my_config.baz_count, 42,); +/// } +/// ``` +/// +/// # Note: /// Unfortunately associated constants are not currently object-safe, and so this /// extends the object-safe [`ExtensionOptions`] pub trait ConfigExtension: ExtensionOptions { @@ -901,6 +943,8 @@ pub trait ConfigExtension: ExtensionOptions { } /// An object-safe API for storing arbitrary configuration +/// do +/// See [`ConfigExtension`] for user defined configuration pub trait ExtensionOptions: Send + Sync + fmt::Debug + 'static { /// Return `self` as [`Any`] /// From d56110df565f93d20a1682fd4c7b0cbdcf7ead00 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 31 Dec 2024 10:18:10 -0500 Subject: [PATCH 3/6] more docs --- datafusion/common/src/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index c7faeb018737..354f2c3c0e6b 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1152,6 +1152,8 @@ pub trait Visit { /// - ``: Default value matching the field type like `42`. /// /// # Example +/// See also a full example on the [`ConfigExtension`] documentation +/// /// ``` /// use datafusion_common::extensions_options; /// From 01995c96a52de616df0c243a747e8383f67e7749 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 31 Dec 2024 10:18:47 -0500 Subject: [PATCH 4/6] Remove update --- datafusion-examples/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index ae404ecce324..23cf8830e36d 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -53,7 +53,6 @@ cargo run --example dataframe - [`analyzer_rule.rs`](examples/analyzer_rule.rs): Use a custom AnalyzerRule to change a query's semantics (row level access control) - [`catalog.rs`](examples/catalog.rs): Register the table into a custom catalog - [`composed_extension_codec`](examples/composed_extension_codec.rs): Example of using multiple extension codecs for serialization / deserialization -- [`config_extensions`](examples/composed_extension_codec.rs): Pass custom options with `ConfigOptions` - [`csv_sql_streaming.rs`](examples/csv_sql_streaming.rs): Build and run a streaming query plan from a SQL statement against a local CSV file - [`custom_datasource.rs`](examples/custom_datasource.rs): Run queries against a custom datasource (TableProvider) - [`custom_file_format.rs`](examples/custom_file_format.rs): Write data to a custom file format From 83881c059ec7f5ea8d9d2d77ce3d91e5dab61773 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 31 Dec 2024 15:45:58 -0500 Subject: [PATCH 5/6] clippy --- datafusion/common/src/config.rs | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 354f2c3c0e6b..3e1f4397152b 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -901,8 +901,8 @@ impl ConfigOptions { /// config::ConfigExtension, extensions_options, /// config::ConfigOptions, /// }; -/// // Define a new configuration struct using the `extensions_options` macro -/// extensions_options! { +/// // Define a new configuration struct using the `extensions_options` macro +/// extensions_options! { /// /// My own config options. /// pub struct MyConfig { /// /// Should "foo" be replaced by "bar"? @@ -911,25 +911,23 @@ impl ConfigOptions { /// /// How many "baz" should be created? /// pub baz_count: usize, default = 1337 /// } -/// } +/// } /// -/// impl ConfigExtension for MyConfig { +/// impl ConfigExtension for MyConfig { /// const PREFIX: &'static str = "my_config"; -/// } +/// } /// -/// fn main() { -/// // set up config struct and register extension -/// let mut config = ConfigOptions::default(); -/// config.extensions.insert(MyConfig::default()); +/// // set up config struct and register extension +/// let mut config = ConfigOptions::default(); +/// config.extensions.insert(MyConfig::default()); /// -/// // overwrite config default -/// config.set("my_config.baz_count", "42").unwrap(); +/// // overwrite config default +/// config.set("my_config.baz_count", "42").unwrap(); /// -/// // check config state -/// let my_config = config.extensions.get::().unwrap(); -/// assert!(my_config.foo_to_bar,); -/// assert_eq!(my_config.baz_count, 42,); -/// } +/// // check config state +/// let my_config = config.extensions.get::().unwrap(); +/// assert!(my_config.foo_to_bar,); +/// assert_eq!(my_config.baz_count, 42,); /// ``` /// /// # Note: From c433575f89583ffd0884d5a7578f378c1d77579e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 1 Jan 2025 12:44:41 -0500 Subject: [PATCH 6/6] Fix issue with ExtensionsOptions docs --- datafusion/common/src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 8810ee635567..8d2742aaafe5 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -946,8 +946,8 @@ pub trait ConfigExtension: ExtensionOptions { const PREFIX: &'static str; } -/// An object-safe API for storing arbitrary configuration -/// do +/// An object-safe API for storing arbitrary configuration. +/// /// See [`ConfigExtension`] for user defined configuration pub trait ExtensionOptions: Send + Sync + fmt::Debug + 'static { /// Return `self` as [`Any`]