Upgrade to datafusion 38 (#691)

* chore: upgrade datafusion Deps

Ref #690

* update concat and concat_ws to use datafusion_functions

Moved in apache/datafusion#10089

* feat: upgrade functions.rs

Upstream is continuing it's migration to UDFs.

Ref apache/datafusion#10098
Ref apache/datafusion#10372

* fix ScalarUDF import

* feat: remove deprecated suppors_filter_pushdown and impl supports_filters_pushdown

Deprecated function removed in apache/datafusion#9923

* use `unnest_columns_with_options` instead of deprecated `unnest_column_with_option`

* remove ScalarFunction wrappers

These relied on upstream BuiltinScalarFunction, which are now removed.

Ref apache/datafusion#10098

* update dataframe `test_describe`

`null_count` was fixed upstream.

Ref apache/datafusion#10260

* remove PyDFField and related methods

DFField was removed upstream.

Ref: apache/datafusion#9595

* bump `datafusion-python` package version to 38.0.0

* re-implement `PyExpr::column_name`

The previous implementation relied on `DFField` which was removed upstream.

Ref: apache/datafusion#9595

Cargo.toml

@@ -17,7 +17,7 @@
 
 [package]
 name = "datafusion-python"
-version = "37.1.0"
+version = "38.0.0"
 homepage = "https://datafusion.apache.org/python"
 repository = "https://github.com/apache/datafusion-python"
 authors = ["Apache DataFusion <[email protected]>"]
@@ -37,13 +37,13 @@ substrait = ["dep:datafusion-substrait"]
 tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] }
-datafusion = { version = "37.1.0", features = ["pyarrow", "avro", "unicode_expressions"] }
-datafusion-common = { version = "37.1.0", features = ["pyarrow"] }
-datafusion-expr = "37.1.0"
-datafusion-functions-array = "37.1.0"
-datafusion-optimizer = "37.1.0"
-datafusion-sql = "37.1.0"
-datafusion-substrait = { version = "37.1.0", optional = true }
+datafusion = { version = "38.0.0", features = ["pyarrow", "avro", "unicode_expressions"] }
+datafusion-common = { version = "38.0.0", features = ["pyarrow"] }
+datafusion-expr = "38.0.0"
+datafusion-functions-array = "38.0.0"
+datafusion-optimizer = "38.0.0"
+datafusion-sql = "38.0.0"
+datafusion-substrait = { version = "38.0.0", optional = true }
 prost = "0.12"
 prost-types = "0.12"
 uuid = { version = "1.8", features = ["v4"] }