From fe5e8e84245adbdc16a4133b38345a6b67016d30 Mon Sep 17 00:00:00 2001 From: keyonghan <54558023+keyonghan@users.noreply.github.com> Date: Wed, 25 Oct 2023 08:28:21 -0700 Subject: [PATCH 1/7] Skip backfiller for recently created new targets (#3190) Fixes https://github.com/flutter/flutter/issues/136487 Context: https://github.com/flutter/flutter/issues/136487#issuecomment-1778101873 --- .../scheduler/batch_backfiller.dart | 14 ++++++++++---- .../scheduler/batch_backfiller_test.dart | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart index 913055b1c..0c80599a3 100644 --- a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart +++ b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart @@ -81,8 +81,8 @@ class BatchBackfiller extends RequestHandler { continue; } final FullTask? backfillTask = _backfillTask(target, taskColumn); - final int priority = backfillPriority(taskColumn.map((e) => e.task).toList(), BatchPolicy.kBatchSize); - if (backfillTask != null) { + final int? priority = backfillPriority(taskColumn.map((e) => e.task).toList()); + if (priority != null && backfillTask != null) { backfill.add(Tuple(target, backfillTask, priority)); } } @@ -200,10 +200,16 @@ class BatchBackfiller extends RequestHandler { /// Returns priority for back filled targets. /// + /// Skips scheduling newly created targets whose available entries are + /// less than `BatchPolicy.kBatchSize`. + /// /// Uses a higher priority if there is an earlier failed build. Otherwise, /// uses default `LuciBuildService.kBackfillPriority` - int backfillPriority(List tasks, int pastTaskNumber) { - if (shouldRerunPriority(tasks, pastTaskNumber)) { + int? backfillPriority(List tasks) { + if (tasks.length < BatchPolicy.kBatchSize) { + return null; + } + if (shouldRerunPriority(tasks, BatchPolicy.kBatchSize)) { return LuciBuildService.kRerunPriority; } return LuciBuildService.kBackfillPriority; diff --git a/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart b/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart index f1617a381..7f81f5f6a 100644 --- a/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart +++ b/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart @@ -125,6 +125,16 @@ void main() { expect(scheduleBuildRequest.priority, LuciBuildService.kBackfillPriority); }); + test('does not backfill targets when number of available tasks is less than BatchPolicy.kBatchSize', () async { + final List scheduleA = [ + // Linux_android A + generateTask(1, name: 'Linux_android A', status: Task.statusNew), + ]; + db.addOnQuery((Iterable results) => scheduleA); + await tester.get(handler); + expect(pubsub.messages.length, 0); + }); + test('backfills earlier failed task with higher priority', () async { final List allGray = [ generateTask(1, name: 'Linux_android A', status: Task.statusNew), @@ -244,15 +254,19 @@ void main() { final List scheduleA = [ // Linux_android A generateTask(1, name: 'Linux_android A', status: Task.statusNew), + generateTask(2, name: 'Linux_android A', status: Task.statusNew), // Linux_android B generateTask(1, name: 'Linux_android B', status: Task.statusNew), + generateTask(2, name: 'Linux_android B', status: Task.statusNew), // Linux_android C generateTask(1, name: 'Linux_android C', status: Task.statusNew), + generateTask(2, name: 'Linux_android C', status: Task.statusNew), ]; db.addOnQuery((Iterable results) => scheduleA); await tester.get(handler); expect(pubsub.messages.length, 2); }); + group('getFilteredBackfill', () { test('backfills high priorty targets first', () async { final List> backfill = >[ From adca9513706292705a3b17c309f3d234fa1e25c2 Mon Sep 17 00:00:00 2001 From: keyonghan <54558023+keyonghan@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:56:22 -0700 Subject: [PATCH 2/7] Add README for cipd packages (#3191) https://github.com/flutter/flutter/issues/137282 --- cipd_packages/README.md | 77 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 cipd_packages/README.md diff --git a/cipd_packages/README.md b/cipd_packages/README.md new file mode 100644 index 000000000..9596db035 --- /dev/null +++ b/cipd_packages/README.md @@ -0,0 +1,77 @@ +# Flutter CIPD Packages + +This folder contains auto built/deployed public [CIPD packages](https://chrome-infra-packages.appspot.com/p/flutter) +used by Flutter CI. These packages will be consumed from tests and auto cached to boost future runs. + +Please follow these steps to add a new package. + +## Creating the package build file + +Create a new sub directory under this folder named with the new package. Add a `build.sh` script for Linux/Mac or +`build.bat` for Windows, and put it under a child dir `tool`. This is the main script to build this package. Add +necessary lib files if needed. + +Note: + +1) we do not archive the binaries in the repo, and will auto build package artifacts and upload to CIPD via the bot. +2) please add a code owner entry under section `cipd packages` in [CODEOWNERS](https://github.com/flutter/cocoon/blob/main/CODEOWNERS) file. + +## Add an entry to cocoon .ci.yaml + +This is to enable the auto build and upload. Please follow the following example format: +```yaml + - name: Mac + recipe: cocoon/cipd + bringup: true + properties: + script: cipd_packages//tool/build.sh + cipd_name: flutter//mac-amd64 # Use mac-arm64 for arm64 version. + device_type: none + runIf: + - cipd_packages//** + - .ci.yaml +``` + +Start with `bringup: true`, same as any other regular new target. Once validated in CI, remove it to enable running +in the `prod` environment so that artifacts are to be uploaded to CIPD. + +Note: with `bringup: true`, the target will be executed in a staging environment and it validates only the logic +and will not upload to CIPD. + +## Adding a reference to the CIPD package + +Until this step, artifacts are being uploaded to CIPD whenever a new commit is merged. It is useful to add a reference +to the package, so that we can use the reference in the CI recipe. This way we won’t need to change the recipe +whenever we update the package. + +Googlers have default access to add a reference to a package via: +```sh +cipd set-ref flutter/PackageName/mac-amd64 -ref Reference -version InstanceID +``` + +* Reference: e.g. major release versions. If not specified, `latest` will be used based on the latest package instance. +* InstaneID: this can be obtained from the package page, e.g. [ruby](https://chrome-infra-packages.appspot.com/p/flutter/ruby/mac-amd64/+/TyvPskvefNRkTDmiDcwRHrdL_a2FQE_4wBojOqhxdtYC). + +Note: for non-Googler contributors, please file an [infra bug](https://github.com/flutter/flutter/issues/new?assignees=&labels=team-infra&projects=&template=6_infrastructure.yml) to make a reference request. + +## Supporting packages download from CI recipe + +Example CL: [51547 ](https://flutter-review.googlesource.com/c/recipes/+/51547) + +Refer to [CONTRIBUTING.md](https://flutter.googlesource.com/recipes/+/refs/heads/main/CONTRIBUTING.md) on how to +contribute to recipes repository. + +## Adding/updating package dependency from .ci.yaml from different repositories + +This is the last step to enable the package usage in the real CI. Add or update the new package to either the platform level or +target level entries for your targeted repository: +``` yaml +dependencies: >- + [ + {"dependency": "chrome_and_driver", "version": "Reference"}, + ] +``` + +Note: use the `Reference` created above. + +More details about configuration setup can be found in [CI_YAML.md](https://github.com/flutter/cocoon/blob/main/CI_YAML.md). \ No newline at end of file From cb8253dd81a311091347d2ad5e4d3baeea77689b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:28:04 +0000 Subject: [PATCH 3/7] Bump test from 1.24.8 to 1.24.9 in /app_dart (#3192) Bumps [test](https://github.com/dart-lang/test/tree/master/pkgs) from 1.24.8 to 1.24.9.
Release notes

Sourced from test's releases.

package:test v1.24.9

  • Update the vm_service constraint to allow version 13.x.
Commits
  • 78ce945 release to support the latest vm_service (#2131)
  • 472874a Update package:vm_service to >=6.0.0 <14.0.0 (#2130)
  • def41cc Tweak dart2js source map rewriting (#2128)
  • f196d18 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test_core (#2125)
  • 9db7e03 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test (#2124)
  • 5420745 Prepare for the latest dart_flutter_team_lints (#2127)
  • 6ac4ff9 Move more js_util usage to the dom library (#2121)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=test&package-manager=pub&previous-version=1.24.8&new-version=1.24.9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- app_dart/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app_dart/pubspec.yaml b/app_dart/pubspec.yaml index 46ba5c79d..cbccc860b 100644 --- a/app_dart/pubspec.yaml +++ b/app_dart/pubspec.yaml @@ -49,7 +49,7 @@ dev_dependencies: json_serializable: 6.7.1 mockito: 5.4.2 platform: 3.1.3 - test: 1.24.8 + test: 1.24.9 builders: json_serializable: 3.3.0 From 3b305770221371f348dee76fdb6a4500e63d653c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:43:05 +0000 Subject: [PATCH 4/7] Bump test from 1.24.8 to 1.24.9 in /cipd_packages/device_doctor (#3193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- Bumps [test](https://github.com/dart-lang/test/tree/master/pkgs) from 1.24.8 to 1.24.9.
Release notes

Sourced from test's releases.

package:test v1.24.9

  • Update the vm_service constraint to allow version 13.x.
Commits
  • 78ce945 release to support the latest vm_service (#2131)
  • 472874a Update package:vm_service to >=6.0.0 <14.0.0 (#2130)
  • def41cc Tweak dart2js source map rewriting (#2128)
  • f196d18 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test_core (#2125)
  • 9db7e03 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test (#2124)
  • 5420745 Prepare for the latest dart_flutter_team_lints (#2127)
  • 6ac4ff9 Move more js_util usage to the dom library (#2121)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=test&package-manager=pub&previous-version=1.24.8&new-version=1.24.9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- cipd_packages/device_doctor/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipd_packages/device_doctor/pubspec.yaml b/cipd_packages/device_doctor/pubspec.yaml index a599244a7..79e3fcf06 100644 --- a/cipd_packages/device_doctor/pubspec.yaml +++ b/cipd_packages/device_doctor/pubspec.yaml @@ -19,4 +19,4 @@ dev_dependencies: build_runner: 2.4.6 fake_async: 1.3.1 mockito: 5.4.2 - test: 1.24.8 + test: 1.24.9 From 4f20b4ae2fe06c29d43ed9f068fd69a37ff3a2c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:47:04 +0000 Subject: [PATCH 5/7] Bump test from 1.24.8 to 1.24.9 in /auto_submit (#3194) Bumps [test](https://github.com/dart-lang/test/tree/master/pkgs) from 1.24.8 to 1.24.9.
Release notes

Sourced from test's releases.

package:test v1.24.9

  • Update the vm_service constraint to allow version 13.x.
Commits
  • 78ce945 release to support the latest vm_service (#2131)
  • 472874a Update package:vm_service to >=6.0.0 <14.0.0 (#2130)
  • def41cc Tweak dart2js source map rewriting (#2128)
  • f196d18 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test_core (#2125)
  • 9db7e03 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test (#2124)
  • 5420745 Prepare for the latest dart_flutter_team_lints (#2127)
  • 6ac4ff9 Move more js_util usage to the dom library (#2121)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=test&package-manager=pub&previous-version=1.24.8&new-version=1.24.9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- auto_submit/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_submit/pubspec.yaml b/auto_submit/pubspec.yaml index bdda14813..b074b7029 100644 --- a/auto_submit/pubspec.yaml +++ b/auto_submit/pubspec.yaml @@ -35,7 +35,7 @@ dev_dependencies: json_serializable: 6.7.1 flutter_lints: 3.0.0 mockito: 5.4.2 - test: 1.24.8 + test: 1.24.9 builders: json_serializable: 3.3.0 From 7f652fd6743a9e16ae3312d33073ba1b61a85fe4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 23:16:21 +0000 Subject: [PATCH 6/7] Bump test from 1.24.8 to 1.24.9 in /cipd_packages/codesign (#3195) Bumps [test](https://github.com/dart-lang/test/tree/master/pkgs) from 1.24.8 to 1.24.9.
Release notes

Sourced from test's releases.

package:test v1.24.9

  • Update the vm_service constraint to allow version 13.x.
Commits
  • 78ce945 release to support the latest vm_service (#2131)
  • 472874a Update package:vm_service to >=6.0.0 <14.0.0 (#2130)
  • def41cc Tweak dart2js source map rewriting (#2128)
  • f196d18 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test_core (#2125)
  • 9db7e03 Bump dart_flutter_team_lints from 1.0.0 to 2.1.1 in /pkgs/test (#2124)
  • 5420745 Prepare for the latest dart_flutter_team_lints (#2127)
  • 6ac4ff9 Move more js_util usage to the dom library (#2121)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=test&package-manager=pub&previous-version=1.24.8&new-version=1.24.9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- cipd_packages/codesign/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cipd_packages/codesign/pubspec.yaml b/cipd_packages/codesign/pubspec.yaml index 5f3f6285a..57d6ed3db 100644 --- a/cipd_packages/codesign/pubspec.yaml +++ b/cipd_packages/codesign/pubspec.yaml @@ -8,7 +8,7 @@ environment: dev_dependencies: lints: 3.0.0 - test: 1.24.8 + test: 1.24.9 dependencies: archive: 3.4.6 args: 2.4.2 From c1828515e2e38b221479863270e52fa493bffe1a Mon Sep 17 00:00:00 2001 From: godofredoc Date: Fri, 27 Oct 2023 13:31:44 -0700 Subject: [PATCH 7/7] Remove all third_party apps in recovery mode. (#3197) The filters were incorrect and given that we are alredy getting only the list of third_party apps we are removing the unnecessary filter. Bug: https://github.com/flutter/flutter/issues/135035 --- cipd_packages/device_doctor/lib/src/android_device.dart | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cipd_packages/device_doctor/lib/src/android_device.dart b/cipd_packages/device_doctor/lib/src/android_device.dart index fd8b8ebd2..8d5062b51 100644 --- a/cipd_packages/device_doctor/lib/src/android_device.dart +++ b/cipd_packages/device_doctor/lib/src/android_device.dart @@ -428,11 +428,7 @@ class AndroidDevice implements Device { if (packageMatch != null) { final packageName = packageMatch.group(1); if (packageName != null) { - if (packageName.contains('/data/app/com.example') | - packageName.contains('/data/app/com.yourcompany') | - packageName.contains('flutter')) { - packages.add(packageName); - } + packages.add(packageName); } } });