Merge branch 'ci/backport_master_changes_v5.2' into 'release/v5.2'

Ci/backport master changes v5.2

See merge request espressif/esp-idf!27841

.gitlab/ci/common.yml

@@ -172,6 +172,11 @@ variables:
   # Show ccache statistics if enabled globally
   test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats || true
 
+.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
+  if [ $CI_JOB_STATUS = "failed" ]; then
+    python tools/ci/artifacts_handler.py upload --type logs
+  fi
+
 .before_script:minimal:
   before_script:
     - *common-before_scripts
@@ -198,6 +203,7 @@ variables:
 .after_script:build:ccache:
   after_script:
     - *show_ccache_statistics
+    - *upload_failed_job_log_artifacts
 
 ##############################
 # Git Strategy Job Templates #
@@ -222,7 +228,8 @@ variables:
     else
       git remote add origin "${MIRROR_REPO_URL}"
     fi
-    git fetch origin --no-recurse-submodules
+    # mirror url may fail with authentication issue
+    git fetch origin --no-recurse-submodules || true
   fi
 
   # set remote url to CI_REPOSITORY_URL
@@ -263,10 +270,14 @@ variables:
         git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
         export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA)
       # other pipelines, like the protected branches pipelines
-      else
+      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
         git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
         git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
         export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA)
+      else
+        # pipeline source could be web, scheduler, etc.
+        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
+        export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA)
       fi
     - *git_checkout_fetch_head
     - *common-before_scripts

.gitlab/ci/docs.yml

@@ -196,8 +196,6 @@ build_docs_pdf_prod:
 
 .deploy_docs_template:
   image: $ESP_IDF_DOC_ENV_IMAGE
-  extends:
-    - .before_script:fetch:git_describe
   variables:
     DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
     PYTHONUNBUFFERED: 1

.gitlab/ci/host-test.yml

@@ -325,6 +325,7 @@ test_pytest_linux:
     paths:
       - XUNIT_RESULT.xml
       - pytest_embedded_log/
+      - "**/build*/build_log.txt"
     reports:
       junit: XUNIT_RESULT.xml
     expire_in: 1 week

.gitlab/ci/pre_check.yml

@@ -186,6 +186,7 @@ pipeline_variables:
         echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env
       fi
     - cat pipeline.env
+    - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
   artifacts:
     reports:
       dotenv: pipeline.env

.gitlab/ci/static-code-analysis.yml

@@ -32,7 +32,7 @@ check_pylint:
       if [ -n "$CI_MERGE_REQUEST_IID" ]; then
         export files=$(echo "$GIT_DIFF_OUTPUT" | grep ".py$" | xargs);
       else
-        export files=$(find . -iname "*.py" -print);
+        export files=$(git ls-files "*.py" | xargs);
       fi
     - if [ -z "$files" ]; then echo "No python files found"; exit 0; fi
     - run_cmd pylint --exit-zero --load-plugins=pylint_gitlab --output-format=gitlab-codeclimate:pylint.json $files

.gitlab/ci/target-test.yml

@@ -11,6 +11,8 @@
       paths:
         - .cache/pip
       policy: pull
+  after_script:
+    - python tools/ci/artifacts_handler.py upload --type logs junit_reports
 
 .pytest_template:
   extends:

tools/ci/artifacts_handler.py

@@ -21,6 +21,8 @@ class ArtifactType(str, Enum):
 
     LOGS = 'logs'
     SIZE_REPORTS = 'size_reports'
+    JUNIT_REPORTS = 'junit_reports'
+    MODIFIED_FILES_AND_COMPONENTS_REPORT = 'modified_files_and_components_report'
 
 
 TYPE_PATTERNS_DICT = {
@@ -48,6 +50,12 @@ class ArtifactType(str, Enum):
         '**/build*/size.json',
         'size_info.txt',
     ],
+    ArtifactType.JUNIT_REPORTS: [
+        'XUNIT_RESULT.xml',
+    ],
+    ArtifactType.MODIFIED_FILES_AND_COMPONENTS_REPORT: [
+        'pipeline.env',
+    ],
 }