From 74be84e1f7f3a0d25578de6ddabf02372477b0a2 Mon Sep 17 00:00:00 2001 From: Beaux Sharifi Date: Wed, 3 Jul 2024 12:53:33 -0700 Subject: [PATCH] Update version to 6.3.0 and supporting docs for release. --- CHANGELOG | 9 +++++++++ README.md | 10 +++++----- version.sbt | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index b7d8724a..fda7dac2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -16,6 +16,15 @@ # spark-redshift Changelog +## 6.3.0 (2024-07-03) +- Validates connector tests pass with Spark releases 3.4.3 and 3.5.1 [Beaux Sharifi] +- Adds support for three-part table names to allow connector to query Redshift data sharing tables (#153) [Prashant Singh, Beaux Sharifi] +- Corrects mapping of Spark ShortType to use Redshift SMALLINT instead of INTEGER to better match expected data size (#152) [Akira Ajisaka, Ruei Huang] +- Adds pushdown of toprettystring() function to support df.show() operations used by Spark 3.5 [Beaux Sharifi] +- Corrects pushdown of inner joins with no join condition to result in a cross join instead of an invalid inner join [Beaux Sharifi] +- Corrects pushdown when casting null boolean values to strings to result in the value null instead of the string "null". [Beaux Sharifi] +- Upgrades Redshift JDBC driver to the latest available version 2.1.0.29. [Beaux Sharifi] + ## 6.2.0 (2024-01-12) - Validates support for Spark 3.3.4 and Spark 3.4.2 - Upgrades Redshift JDBC driver to version 2.1.0.24 diff --git a/README.md b/README.md index 500fc3aa..9eff0995 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ You may use this library in your applications with the following dependency info spark-submit \ --deploy-mode cluster \ --master yarn \ - --packages com.amazon.redshift:redshift-jdbc42:2.1.0.29,org.apache.spark:spark-avro_2.12:3.5.1,io.github.spark-redshift-community:spark-redshift_2.12:6.2.0-spark_3.5 \ + --packages com.amazon.redshift:redshift-jdbc42:2.1.0.29,org.apache.spark:spark-avro_2.12:3.5.1,io.github.spark-redshift-community:spark-redshift_2.12:6.3.0-spark_3.5 \ my_script.py ``` @@ -114,14 +114,14 @@ You may use this library in your applications with the following dependency info io.github.spark-redshift-community spark-redshift_2.12 - 6.2.0-spark_3.5 + 6.3.0-spark_3.5 ``` - **In SBT**: ```SBT - libraryDependencies += "io.github.spark-redshift-community" %% "spark-redshift_2.12" % "6.2.0-spark_3.5" + libraryDependencies += "io.github.spark-redshift-community" %% "spark-redshift_2.12" % "6.3.0-spark_3.5" ``` @@ -816,7 +816,7 @@ for more information.

"" An identifier to include in the query group set when running queries with the connector. Should be 100 or fewer characters and all characters must be valid unicodeIdentifierParts. Characters in excess of 100 will be trimmed. - When running a query with the connector a json formatted string will be set as the query group (for example `{"spark-redshift-connector":{"svc":"","ver":"6.2.0-spark_3.5","op":"Read","lbl":"","tid":""}}`). + When running a query with the connector a json formatted string will be set as the query group (for example `{"spark-redshift-connector":{"svc":"","ver":"6.3.0-spark_3.5","op":"Read","lbl":"","tid":""}}`). This option will be substituted for the value of the `lbl` key. @@ -924,7 +924,7 @@ SET spark.datasource.redshift.community.autopushdown.lazyMode=false ### trace_id A new tracing identifier field that is added to the existing `label` parameter. When set, the provided string value will be used as part of label. Otherwise, it will default to the Spark application identifier. For example: -`{"spark-redshift-connector":{"svc":"","ver":"6.2.0-spark_3.5","op":"Read","lbl":"","tid":"..."}}`) +`{"spark-redshift-connector":{"svc":"","ver":"6.3.0-spark_3.5","op":"Read","lbl":"","tid":"..."}}`) To set the value, run the following command: ```sparksql diff --git a/version.sbt b/version.sbt index b725a4af..34b535f8 100644 --- a/version.sbt +++ b/version.sbt @@ -14,4 +14,4 @@ * limitations under the License. */ -ThisBuild / version := "6.2.0" +ThisBuild / version := "6.3.0"