Merge branch 'apache:master' into master

.asf.yaml

@@ -54,6 +54,7 @@ github:
           - P0 Regression (Doris Regression)
           - P1 Regression (Doris Regression)
           - External Regression (Doris External Regression)
+          - cloud_p1 (Doris Cloud Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
@@ -62,6 +63,8 @@ github:
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)
           - Need_2_Approval
+          - Cloud UT (Doris Cloud UT)
+          - performance (Doris Performance)
 
       required_pull_request_reviews:
         dismiss_stale_reviews: true

.clang-tidy

@@ -23,6 +23,8 @@ Checks: |
   -readability-inconsistent-declaration-parameter-name,
   -readability-isolate-declaration,
   -readability-named-parameter,
+  -readability-avoid-const-params-in-decls,
+  -readability-convert-member-functions-to-static,
   portability-simd-intrinsics,
   performance-type-promotion-in-math-fn,
   performance-faster-string-find,

.github/workflows/build-extension.yml

@@ -61,7 +61,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: adopt
-          java-version: '8'
+          java-version: '17'
 
       - name: Setup thrift
         run: |
@@ -84,39 +84,39 @@ jobs:
       - name: Build broker
         run: |
           cd fs_brokers/apache_hdfs_broker/ && /bin/bash build.sh
-  build-docs:
-    name: Build Documents
-    needs: changes
-    if: ${{ needs.changes.outputs.docs_changes == 'true' }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout ${{ github.ref }}
-        uses: actions/checkout@v3
+  # build-docs:
+  #   name: Build Documents
+  #   needs: changes
+  #   if: ${{ needs.changes.outputs.docs_changes == 'true' }}
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout ${{ github.ref }}
+  #       uses: actions/checkout@v3
 
-      - name: Build docs
-        run: |
-          cd docs && /bin/bash build_help_zip.sh
-      - name: Build
-        run: |
-          git clone https://github.com/apache/doris-website.git website
-          cd website
-          echo "[\"current\"]" > versions.json 
-          mkdir -p docs
-          cp -R ../docs/en/docs/* docs/
-          cp -R ../docs/sidebars.json sidebars.json
-          mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs/current
-          cp -R ../docs/zh-CN/docs/* i18n/zh-CN/docusaurus-plugin-content-docs/current/
-          cp -R ../docs/dev.json i18n/zh-CN/docusaurus-plugin-content-docs/current.json
+  #     - name: Build docs
+  #       run: |
+  #         cd docs && /bin/bash build_help_zip.sh
+  #     - name: Build
+  #       run: |
+  #         git clone https://github.com/apache/doris-website.git website
+  #         cd website
+  #         echo "[\"current\"]" > versions.json 
+  #         mkdir -p docs
+  #         cp -R ../docs/en/docs/* docs/
+  #         cp -R ../docs/sidebars.json sidebars.json
+  #         mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs/current
+  #         cp -R ../docs/zh-CN/docs/* i18n/zh-CN/docusaurus-plugin-content-docs/current/
+  #         cp -R ../docs/dev.json i18n/zh-CN/docusaurus-plugin-content-docs/current.json
 
-          mkdir -p community
-          cp -R ../docs/en/community/* community/
-          mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
-          cp -R ../docs/zh-CN/community/* i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
-          cp -R ../docs/sidebarsCommunity.json .
+  #         mkdir -p community
+  #         cp -R ../docs/en/community/* community/
+  #         mkdir -p i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
+  #         cp -R ../docs/zh-CN/community/* i18n/zh-CN/docusaurus-plugin-content-docs-community/current/
+  #         cp -R ../docs/sidebarsCommunity.json .
 
-          cp -R ../docs/images static/
-          npm install -g yarn
-          yarn cache clean
-          yarn && yarn build
-          cd ../
-          rm -rf website
+  #         cp -R ../docs/images static/
+  #         npm install -g yarn
+  #         yarn cache clean
+  #         yarn && yarn build
+  #         cd ../
+  #         rm -rf website

.github/workflows/clang-format.yml

@@ -17,7 +17,7 @@
 #
 
 ---
-name: BE Code Formatter
+name: Code Formatter
 
 on: [push, pull_request_target]
 
@@ -52,9 +52,11 @@ jobs:
         id: filter
         with:
           filters: |
-            be_changes:
+            changes:
               - 'be/src/**'
               - 'be/test/**'
+              - 'cloud/src/**'
+              - 'cloud/test/**'
 
       - name: Checkout clang-format-lint-action
         run: |
@@ -66,14 +68,14 @@ jobs:
           popd &>/dev/null
 
       - name: "Format it!"
-        if: ${{ steps.filter.outputs.be_changes == 'true' }}
+        if: ${{ steps.filter.outputs.changes == 'true' }}
         uses: ./.github/actions/clang-format-lint-action
-        id: be_clang_format
+        id: clang_format
         with:
-          source: "be/src be/test"
+          source: "be/src be/test cloud/src cloud/test"
           clangFormatVersion: 16
           inplace: False
 
       - name: Ignore it!
-        if: ${{ steps.filter.outputs.be_changes == 'false' }}
+        if: ${{ steps.filter.outputs.changes == 'false' }}
         run: echo 'No need to check.'

.github/workflows/comment-to-trigger-teamcity.yml

@@ -148,45 +148,64 @@ jobs:
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_fe_ut }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "feut" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_fe_ut }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "feut" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch is not in (master, branch-2.0), skip run feut"
+        fi
 
 
     - name: "Trigger or Skip beut"
       if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_be_ut }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "beut" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ||
+              "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_be_ut }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "beut" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch is not in (master, branch-2.0), skip run beut"
+        fi
 
     - name: "Trigger or Skip cloudut"
       if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloudut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_cloud_ut }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "cloudut" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_cloud_ut }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "cloudut" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch is not master, skip run cloudut"
+        fi
 
     - name: "Trigger or Skip compile"
       if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
+        if [[ "${{ steps.changes.outputs.changed_p0 }}" == "true" ]] || [[ "${{ steps.changes.outputs.changed_cloud_p0 }}" == "true" ]]; then
+          changed="true"
+        else
+          changed="false"
+        fi
         trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_p0 }}" \
+          "${changed}" \
           "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
           "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
           "compile" \

.github/workflows/pr-approve-status.yml

@@ -19,42 +19,25 @@ name: Need_2_Approval
 
 on:
   pull_request_review:
-    types: [submitted]
+    types: [ submitted ]
+
 jobs:
   Need_2_Approval:
     runs-on: ubuntu-latest
-    timeout-minutes: 3
+    timeout-minutes: 5
     steps:
       - uses: actions/checkout@v3
-      - run: |
-          pr_num=${{ github.event.pull_request.number }}
-          echo $pr_num
-          if [ -z "$pr_num" ]; then
-              echo "PR number is not set"
-              exit 1
-          fi
-          response=$(curl -s  -H "Authorization: token ${{ secrets.GITHUB_TOKEN }} "  "https://api.github.com/repos/apache/doris/pulls/${pr_num}/reviews?per_page=100")
-          # shellcheck disable=SC2207
-          reviewers=($(echo $response | jq -r '.[] | .user.login'))
-          # shellcheck disable=SC2207
-          statuses=($(echo $response | jq -r '.[] | .state'))
-          echo "${reviewers[@]}"
-          echo "${statuses[@]}"
-          approves=()
-          reviewers_unique=()
-          for ((i=${#reviewers[@]}-1;i>=0;i--)); do
-            if ! echo "${reviewers_unique[@]}" | grep -q -w "${reviewers[$i]}" && [ "${statuses[$i]}" != "COMMENTED" ]; then
-                  reviewers_unique+=( "${reviewers[$i]}" )
-                  if [ "${statuses[$i]}" == "APPROVED" ]; then
-                      approves+=( "${reviewers[$i]}" )
-                  fi
-              fi
-          done
-          echo "${approves[@]}"
-          if [ ${#approves[@]} -lt 2 ]; then
-              echo "PR ${pr_num} has not been approved by at least 2 reviewers"
-              # shellcheck disable=SC2242
-              exit 1
-          fi
-          echo "Thanks for your contribution to Doris."
-          exit 0
+      - name: Install Python dependencies
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'  # Adjust if needed
+      - name: Install match library
+        run: |
+          pip install --upgrade pip 
+          pip install match
+          pip install requests
+      - name: Run Python script
+        run: |
+          python tools/maintainers/check_review.py  ${{ github.event.pull_request.number }} ${{secrets.GITHUB_TOKEN}}
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

LICENSE.txt

@@ -718,3 +718,11 @@ LICENSE file in the root directory of this source tree. An additional grant
 of patent rights can be found in the PATENTS file in the same directory.
 
 ----------------------------------------------------------------------------------
+
+me.bechberger.ap-loader-all: Apache 2.0 license
+
+Apache 2.0, Copyright 2023 SAP SE or an SAP affiliate company, Johannes Bechberger and ap-loader contributors
+
+This project is maintained by the SapMachine team at SAP SE
+
+----------------------------------------------------------------------------------

README.md

@@ -35,10 +35,9 @@ Apache Doris is an easy-to-use, high-performance and real-time analytical databa
 
 All this makes Apache Doris an ideal tool for scenarios including report analysis, ad-hoc query, unified data warehouse, and data lake query acceleration. On Apache Doris, users can build various applications, such as user behavior analysis, AB test platform, log retrieval analysis, user portrait analysis, and order analysis.
 
-🎉 Version 2.1.0 released now. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.1.0) here. The 2.1 verison delivers exceptional performance with 100% higher out-of-the-box queries proven by TPC-DS 1TB tests, enhanced data lake analytics that are 4-6 times speedier than Trino and Spark, solid support for semi-structured data analysis with new Variant types and suite of analytical functions, asynchronous materialized views for query acceleration, optimized real-time writing at scale, and better workload management with stability and runtime SQL resource tracking.
+Please visit our [official download page](https://doris.apache.org/download/) to get the latest release version.
 
-
-🎉 Version 2.0.6 is now released ! This fully evolved and stable release is ready for all users to upgrade. Check out the 🔗[Release Notes](https://doris.apache.org/docs/releasenotes/release-2.0.6) here. 
+The current stable version is the 2.0.x series, and the latest version is the 2.1.x series. For production, it is recommended to use the latest version of the 2.0.x series. And if used for POC or testing, it is recommended to use the latest version of the 2.1.x series.
 
 👀 Have a look at the 🔗[Official Website](https://doris.apache.org/) for a comprehensive list of Apache Doris's core features, blogs and user cases.
 

be/CMakeLists.txt

@@ -72,6 +72,7 @@ option(USE_LIBCPP "Use libc++" OFF)
 option(USE_MEM_TRACKER, "Use memory tracker" ON)
 option(USE_UNWIND "Use libunwind" ON)
 option(USE_JEMALLOC "Use jemalloc" ON)
+option(USE_JEMALLOC_HOOK "Use jemalloc hook" ON)
 if (OS_MACOSX)
     set(GLIBC_COMPATIBILITY OFF)
     set(USE_LIBCPP ON)
@@ -87,6 +88,7 @@ message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}")
 message(STATUS "USE_LIBCPP is ${USE_LIBCPP}")
 message(STATUS "USE_MEM_TRACKER is ${USE_MEM_TRACKER}")
 message(STATUS "USE_JEMALLOC is ${USE_JEMALLOC}")
+message(STATUS "USE_JEMALLOC_HOOK is ${USE_JEMALLOC_HOOK}")
 message(STATUS "USE_UNWIND is ${USE_UNWIND}")
 message(STATUS "ENABLE_PCH is ${ENABLE_PCH}")
 
@@ -280,9 +282,13 @@ if (COMPILER_CLANG)
                         -Wunused-volatile-lvalue
                         -Wunused-template
                         -Wunused-member-function
-                        -Wunused-macros)
-    add_compile_options(-Wno-vla-extension)
-    add_compile_options(-Wno-gnu-statement-expression)
+                        -Wunused-macros
+                        -Wconversion)
+    add_compile_options( -Wno-gnu-statement-expression
+                        -Wno-implicit-float-conversion
+                        -Wno-implicit-int-conversion
+                        -Wno-sign-conversion
+                        -Wno-shorten-64-to-32)
     if (USE_LIBCPP)
         add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>)
         if (NOT OS_MACOSX)
@@ -324,6 +330,10 @@ if (WITH_MYSQL)
     add_definitions(-DDORIS_WITH_MYSQL)
 endif()
 
+if (ENABLE_INJECTION_POINT)
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DENABLE_INJECTION_POINT")
+endif()
+
 # Enable memory tracker, which allows BE to limit the memory of tasks such as query, load,
 # and compaction,and observe the memory of BE through be_ip:http_port/MemTracker.
 # Adding the option `USE_MEM_TRACKER=OFF sh build.sh` when compiling can turn off the memory tracker,
@@ -337,6 +347,9 @@ endif()
 if (USE_JEMALLOC)
     add_definitions(-DUSE_JEMALLOC)
 endif()
+if (USE_JEMALLOC_HOOK)
+    add_definitions(-DUSE_JEMALLOC_HOOK)
+endif()
 
 # Compile with libunwind
 if (USE_UNWIND)

be/cmake/thirdparty.cmake

@@ -150,9 +150,7 @@ add_thirdparty(krb5)
 add_thirdparty(com_err)
 add_thirdparty(k5crypto)
 add_thirdparty(gssapi_krb5)
-add_thirdparty(dragonbox_to_chars LIB64)
 add_thirdparty(streamvbyte LIB64)
-target_include_directories(dragonbox_to_chars INTERFACE "${THIRDPARTY_DIR}/include/dragonbox-1.1.3")
 
 if (OS_MACOSX)
     add_thirdparty(bfd)