Optimization of the algorithm for determining the end of partition processing #27830
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR-check | |
| on: | |
| pull_request_target: | |
| branches: | |
| - 'main' | |
| - 'stable-*' | |
| - 'stream-nb-*' | |
| - '*-stable-*' | |
| - 'dev-*' | |
| types: | |
| - 'opened' | |
| - 'synchronize' | |
| - 'reopened' | |
| - 'labeled' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-running-allowed: | |
| if: ${{vars.CHECKS_SWITCH != '' && fromJSON(vars.CHECKS_SWITCH).pr_check == true}} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| result: ${{ steps.check-ownership-membership.outputs.result == 'true' && steps.check-is-mergeable.outputs.result == 'true' }} | |
| commit_sha: ${{ steps.check-is-mergeable.outputs.commit_sha }} | |
| steps: | |
| - name: Reset integrated status | |
| run: | | |
| curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \ | |
| -d '{"state":"pending","description":"Waiting for relevant checks to complete","context":"checks_integrated"}' | |
| - name: Check if running tests is allowed | |
| id: check-ownership-membership | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| script: | | |
| const labels = context.payload.pull_request.labels; | |
| const okToTestLabel = labels.find( | |
| label => label.name == 'ok-to-test' | |
| ); | |
| console.log("okToTestLabel=%o", okToTestLabel !== undefined); | |
| if (okToTestLabel !== undefined) { | |
| return true; | |
| } | |
| // This is used primarily in forks. Repository owner | |
| // should be allowed to run anything. | |
| const userLogin = context.payload.pull_request.user.login; | |
| // How to interpret membership status code: | |
| // https://docs.github.com/rest/collaborators/collaborators#check-if-a-user-is-a-repository-collaborator | |
| const isRepoCollaborator = async function () { | |
| try { | |
| const response = await github.rest.repos.checkCollaborator({ | |
| owner: context.payload.repository.owner.login, | |
| repo: context.payload.repository.name, | |
| username: userLogin, | |
| }); | |
| return response.status == 204; | |
| } catch (error) { | |
| if (error.status && error.status == 404) { | |
| return false; | |
| } | |
| throw error; | |
| } | |
| } | |
| if (context.payload.repository.owner.login == userLogin) { | |
| console.log("You are the repository owner!"); | |
| return true; | |
| } | |
| if (await isRepoCollaborator()) { | |
| console.log("You are a collaborator!"); | |
| return true; | |
| } | |
| return false; | |
| - name: comment-if-waiting-on-ok | |
| if: steps.check-ownership-membership.outputs.result == 'false' && | |
| github.event.action == 'opened' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| let externalContributorLabel = 'external'; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: 'Hi! Thank you for contributing!\nThe tests on this PR will run after a maintainer adds an `ok-to-test` label to this PR manually. Thank you for your patience!' | |
| }); | |
| github.rest.issues.addLabels({ | |
| ...context.repo, | |
| issue_number: context.issue.number, | |
| labels: [externalContributorLabel] | |
| }); | |
| - name: cleanup-test-label | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| let labelsToRemove = ['ok-to-test', 'rebase-and-check']; | |
| const prNumber = context.payload.pull_request.number; | |
| const prLabels = new Set(context.payload.pull_request.labels.map(l => l.name)); | |
| for await (const label of labelsToRemove.filter(l => prLabels.has(l))) { | |
| core.info(`remove label=${label} for pr=${prNumber}`); | |
| try { | |
| const result = await github.rest.issues.removeLabel({ | |
| ...context.repo, | |
| issue_number: prNumber, | |
| name: label | |
| }); | |
| } catch(error) { | |
| // ignore the 404 error that arises | |
| // when the label did not exist for the | |
| // organization member | |
| if (error.status && error.status != 404) { | |
| throw error; | |
| } | |
| } | |
| } | |
| - name: check is mergeable | |
| id: check-is-mergeable | |
| if: steps.check-ownership-membership.outputs.result == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| result-encoding: string | |
| script: | | |
| let pr = context.payload.pull_request; | |
| const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); | |
| const header = `<!-- merge pr=${pr.number} -->\n`; | |
| const fail_msg = header + ':red_circle: Unable to merge your PR into the base branch. ' | |
| + 'Please rebase or merge it with the base branch.' | |
| let i = 0; | |
| while (pr.mergeable == null && i < 60) { | |
| console.log("get pull-request status"); | |
| let result = await github.rest.pulls.get({ | |
| ...context.repo, | |
| pull_number: pr.number | |
| }) | |
| pr = result.data; | |
| if (pr.mergeable == null) { | |
| await delay(5000); | |
| } | |
| i += 1; | |
| } | |
| console.log("pr.mergeable=%o", pr.mergeable); | |
| if (pr.mergeable === null) { | |
| core.setFailed("Unable to check if the PR is mergeable, please re-run the check."); | |
| return false; | |
| } | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo | |
| }); | |
| const commentToUpdate = comments.find(comment => comment.body.startsWith(header)); | |
| if (!pr.mergeable) { | |
| let commentParams = { | |
| ...context.repo, | |
| issue_number: context.issue.number, | |
| body: fail_msg | |
| }; | |
| if (commentToUpdate) { | |
| await github.rest.issues.updateComment({ | |
| ...commentParams, | |
| comment_id: commentToUpdate.id, | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({...commentParams}); | |
| } | |
| core.setFailed("Merge conflict detected"); | |
| return false; | |
| } else if (commentToUpdate) { | |
| await github.rest.issues.deleteComment({ | |
| ...context.repo, | |
| issue_number: context.issue.number, | |
| comment_id: commentToUpdate.id, | |
| }); | |
| } | |
| core.info(`commit_sha=${pr.commit_sha}`); | |
| core.setOutput('commit_sha', pr.merge_commit_sha); | |
| return true; | |
| build_and_test: | |
| needs: | |
| - check-running-allowed | |
| if: needs.check-running-allowed.outputs.result == 'true' && needs.check-running-allowed.outputs.commit_sha != '' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| build_preset: ["relwithdebinfo", "release-asan"] | |
| runs-on: [ self-hosted, auto-provisioned, "${{ format('build-preset-{0}', matrix.build_preset) }}" ] | |
| name: Build and test ${{ matrix.build_preset }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.check-running-allowed.outputs.commit_sha }} | |
| fetch-depth: 2 | |
| - name: Setup ydb access | |
| uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials | |
| with: | |
| ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} | |
| - name: Build and test | |
| uses: ./.github/actions/build_and_test_ya | |
| with: | |
| build_preset: ${{ matrix.build_preset }} | |
| build_target: "ydb/" | |
| increment: true | |
| run_tests: ${{ contains(fromJSON('["relwithdebinfo", "release-asan"]'), matrix.build_preset) }} | |
| test_size: "small,medium" | |
| test_threads: 52 | |
| put_build_results_to_cache: true | |
| additional_ya_make_args: -DDEBUGINFO_LINES_ONLY # we don't need full symbols in CI checks | |
| secs: ${{ format('{{"TESTMO_TOKEN2":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}', | |
| secrets.TESTMO_TOKEN2, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }} | |
| vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}', | |
| vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }} | |
| update_integrated_status: | |
| runs-on: ubuntu-latest | |
| needs: build_and_test | |
| if: always() | |
| steps: | |
| - name: Gather required checks results | |
| shell: bash | |
| run: | | |
| successbuilds=$(curl -L -X GET -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/${{github.repository}}/commits/${{github.event.pull_request.head.sha}}/status | \ | |
| jq -cr '.statuses | .[] | select(.state=="success") | select(.context | (startswith("build_relwithdebinfo") or startswith("build_release-asan") or startswith("test_relwithdebinfo")) ) | .context' | \ | |
| wc -l ) | |
| if [[ $successbuilds == "3" ]];then | |
| integrated_status="success" | |
| else | |
| integrated_status="failure" | |
| fi | |
| curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \ | |
| -d '{"state":"'$integrated_status'","description":"All checks completed","context":"checks_integrated"}' | |