Merge branch 'apache:main' into gayangya/transform_keys_and_values

.github/workflows/check_license.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: license header check
+on:
+  pull_request
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  license-check:
+    name: License Header Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check License Header
+        run: |
+          git fetch --recurse-submodules=no origin main ${{github.event.pull_request.base.sha}}
+          pip install regex
+          cd $GITHUB_WORKSPACE/
+          ./.github/workflows/util/check.sh ${{github.event.pull_request.base.sha}}

.github/workflows/code_style.yml

@@ -14,43 +14,56 @@
 # limitations under the License.
 
 name: Code style checks
-
 on:
-  pull_request
-
+  pull_request:
+    paths:
+      - '.github/workflows/code_style.yml'
+      - 'cpp/**'
+      - 'cpp-ch/**'
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
 
 jobs:
-  formatting-check:
-    name: Formatting Check
+  CPP-format-check:
+    name: CPP Format Check
     runs-on: ubuntu-latest
     strategy:
       matrix:
         path:
-          - check: 'cpp/core'
-            exclude: ''
-          - check: 'cpp/velox'
+          - check: 'cpp'
             exclude: ''
     steps:
       - uses: actions/checkout@v4
-      - name: Run clang-format style check for C/C++ programs.
+      - name: Run clang-format style check for C/C++
         uses: jidicula/[email protected]
         with:
           clang-format-version: '15'
           check-path: ${{ matrix.path['check'] }}
           fallback-style: 'Google' # optional
 
-  license-check:
-    name: License Header Check
+  CMake-format-check:
+    name: CMake Format Check
     runs-on: ubuntu-latest
-
+    container: ubuntu:22.04
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Check License Header
+      - name: Install tools
+        run: |
+          apt update -y
+          apt install git python3-pip -y
+          pip3 install --user cmake-format
+      - uses: actions/checkout@v4
+      - name: Check CMake format
         run: |
-          git fetch --recurse-submodules=no origin main ${{github.event.pull_request.base.sha}}
-          pip install regex
-          dev/check.sh ${{github.event.pull_request.base.sha}}
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          cd $GITHUB_WORKSPACE/
+          fileList=$(find ./cpp ./cpp-ch -name CMakeLists.txt -o -name *.cmake)
+          for file in $fileList; do
+              /github/home/.local/bin/cmake-format --first-comment-is-literal True --in-place $file
+          done
+          if [ -n "$(git status --porcelain)" ]; then
+             echo "Please use cmake-format to format cmake files or apply the below patch."
+             git diff -- '*CMakeLists.txt' '*.cmake'
+             exit 1
+          fi
+          echo "No CMake format issue."

dev/check.sh → .github/workflows/util/check.sh

@@ -15,9 +15,9 @@
 # limitations under the License.
 
 export BASE_COMMIT=$1
-dev/check.py header branch
+./.github/workflows/util/check.py header branch
 if [ $? -ne 0 ]; then
-  dev/check.py header branch --fix
+  ./.github/workflows/util/check.py header branch --fix
   echo -e "\n==== Apply using:"
   echo "patch -p1 \<<EOF"
   git --no-pager diff

dev/excluded_copyright_files.txt → .github/workflows/util/excluded_copyright_files.txt

@@ -1,2 +1,3 @@
 *cpp-ch/local-engine/Storages/ch_parquet/arrow/*
 *dev/*
+*.github/workflows/util/*

backends-clickhouse/pom.xml

@@ -215,6 +215,38 @@
       <version>8.5.9</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-memory-core</artifactId>
+      <version>${arrow.version}</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-buffer</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-vector</artifactId>
+      <version>${arrow.version}</version>
+      <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-common</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>io.netty</groupId>
+          <artifactId>netty-buffer</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
   <build>
@@ -272,6 +304,7 @@
             </includes>
             <excludes>
               <exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/*.scala</exclude>
+              <exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/commands/merge/*.scala</exclude>
               <exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/files/*.scala</exclude>
               <exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/DeltaLog.scala</exclude>
               <exclude>src/main/delta-${delta.binary.version}/org/apache/spark/sql/delta/Snapshot.scala</exclude>

backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala → backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala

@@ -32,7 +32,6 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec
 import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteJobStatsTracker}
 import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter
 import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig
-import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat
 import org.apache.spark.util.{Clock, SerializableConfiguration}
 
 import org.apache.commons.lang3.exception.ExceptionUtils
@@ -139,20 +138,7 @@ class ClickhouseOptimisticTransaction(
           MergeTreeFileFormatWriter.write(
             sparkSession = spark,
             plan = newQueryPlan,
-            fileFormat = new DeltaMergeTreeFileFormat(
-              metadata,
-              tableV2.dataBaseName,
-              tableV2.tableName,
-              ClickhouseSnapshot.genSnapshotId(tableV2.snapshot),
-              tableV2.orderByKeyOption,
-              tableV2.lowCardKeyOption,
-              tableV2.minmaxIndexKeyOption,
-              tableV2.bfIndexKeyOption,
-              tableV2.setIndexKeyOption,
-              tableV2.primaryKeyOption,
-              tableV2.clickhouseTableConfigs,
-              tableV2.partitionColumns
-            ),
+            fileFormat = tableV2.getFileFormat(metadata),
             // formats.
             committer = committer,
             outputSpec = outputSpec,

backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/DeltaLogAdapter.scala → backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaAdapter.scala

@@ -14,10 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.sql.execution.datasources.v2.clickhouse
+package org.apache.spark.sql.delta
 
-import org.apache.spark.sql.delta.{DeltaLog, Snapshot}
-
-object DeltaLogAdapter {
-  def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.unsafeVolatileSnapshot
+object DeltaAdapter extends DeltaAdapterTrait {
+  override def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.snapshot
 }