Skip to content

Commit

Permalink
Merge branch 'flutter:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@ricardoamador
ricardoamador authored Mar 14, 2024
2 parents caefcbf + 64667f6 commit 77d6696
Show file tree
Hide file tree
Showing 16 changed files with 232 additions and 92 deletions.
6 changes: 3 additions & 3 deletions app_dart/lib/src/request_handlers/get_build_status.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,11 @@

import 'dart:async';

import 'package:cocoon_service/cocoon_service.dart';
import 'package:github/github.dart';
import 'package:meta/meta.dart';

import '../../protos.dart' show BuildStatusResponse, EnumBuildStatus;
import '../request_handling/body.dart';
import '../request_handling/request_handler.dart';
import '../service/build_status_provider.dart';
import '../service/datastore.dart';

Expand All @@ -34,7 +33,8 @@ class GetBuildStatus extends RequestHandler<Body> {

Future<BuildStatusResponse> createResponse() async {
final DatastoreService datastore = datastoreProvider(config.db);
final BuildStatusService buildStatusService = buildStatusProvider(datastore);
final FirestoreService firestoreService = await config.createFirestoreService();
final BuildStatusService buildStatusService = buildStatusProvider(datastore, firestoreService);
final String repoName = request!.uri.queryParameters[kRepoParam] ?? 'flutter';
final RepositorySlug slug = RepositorySlug('flutter', repoName);
final BuildStatus status = (await buildStatusService.calculateCumulativeStatus(slug))!;
Expand Down
4 changes: 3 additions & 1 deletion app_dart/lib/src/request_handlers/get_green_commits.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import '../request_handling/request_handler.dart';
import '../service/build_status_provider.dart';
import '../service/config.dart';
import '../service/datastore.dart';
import '../service/firestore.dart';

/// Returns [List<String>] of the commit shas that had all passing tests.
///
Expand Down Expand Up @@ -52,7 +53,8 @@ class GetGreenCommits extends RequestHandler<Body> {
final RepositorySlug slug = RepositorySlug('flutter', repoName);
final String branch = request!.uri.queryParameters[kBranchParam] ?? Config.defaultBranch(slug);
final DatastoreService datastore = datastoreProvider(config.db);
final BuildStatusService buildStatusService = buildStatusProvider(datastore);
final FirestoreService firestoreService = await config.createFirestoreService();
final BuildStatusService buildStatusService = buildStatusProvider(datastore, firestoreService);
final int commitNumber = config.commitNumber;

final List<String?> greenCommits = await buildStatusService
Expand Down
4 changes: 3 additions & 1 deletion app_dart/lib/src/request_handlers/get_status.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import '../request_handling/request_handler.dart';
import '../service/build_status_provider.dart';
import '../service/config.dart';
import '../service/datastore.dart';
import '../service/firestore.dart';

@immutable
class GetStatus extends RequestHandler<Body> {
Expand All @@ -39,7 +40,8 @@ class GetStatus extends RequestHandler<Body> {
final RepositorySlug slug = RepositorySlug('flutter', repoName);
final String branch = request!.uri.queryParameters[kBranchParam] ?? Config.defaultBranch(slug);
final DatastoreService datastore = datastoreProvider(config.db);
final BuildStatusService buildStatusService = buildStatusProvider(datastore);
final FirestoreService firestoreService = await config.createFirestoreService();
final BuildStatusService buildStatusService = buildStatusProvider(datastore, firestoreService);
final KeyHelper keyHelper = config.keyHelper;
final int commitNumber = config.commitNumber;
final int lastCommitTimestamp = await _obtainTimestamp(encodedLastCommitKey, keyHelper, datastore);
Expand Down
3 changes: 2 additions & 1 deletion app_dart/lib/src/request_handlers/push_build_status_to_github.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ class PushBuildStatusToGithub extends ApiRequestHandler<Body> {
final String repository = request!.uri.queryParameters[fullNameRepoParam] ?? 'flutter/flutter';
final RepositorySlug slug = RepositorySlug.full(repository);
final DatastoreService datastore = datastoreProvider(config.db);
final BuildStatusService buildStatusService = buildStatusServiceProvider(datastore);
final FirestoreService firestoreService = await config.createFirestoreService();
final BuildStatusService buildStatusService = buildStatusServiceProvider(datastore, firestoreService);

final BuildStatus status = (await buildStatusService.calculateCumulativeStatus(slug))!;
await _insertBigquery(slug, status.githubStatus, Config.defaultBranch(slug), config);
Expand Down
16 changes: 12 additions & 4 deletions app_dart/lib/src/service/build_status_provider.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import 'dart:async';

import 'package:cocoon_service/cocoon_service.dart';
import 'package:github/github.dart';
import 'package:meta/meta.dart';

Expand All @@ -14,20 +15,27 @@ import '../model/appengine/task.dart';
import 'datastore.dart';

/// Function signature for a [BuildStatusService] provider.
typedef BuildStatusServiceProvider = BuildStatusService Function(DatastoreService datastoreService);
typedef BuildStatusServiceProvider = BuildStatusService Function(
DatastoreService datastoreService,
FirestoreService firestoreService,
);

/// Branches that are used to calculate the tree status.
const Set<String> defaultBranches = <String>{'refs/heads/main', 'refs/heads/master'};

/// Class that calculates the current build status.
class BuildStatusService {
const BuildStatusService(this.datastoreService);
const BuildStatusService(
this.datastoreService,
this.firestoreService,
);

final DatastoreService datastoreService;
final FirestoreService firestoreService;

/// Creates and returns a [DatastoreService] using [db] and [maxEntityGroups].
static BuildStatusService defaultProvider(DatastoreService datastoreService) {
return BuildStatusService(datastoreService);
static BuildStatusService defaultProvider(DatastoreService datastoreService, FirestoreService firestoreService) {
return BuildStatusService(datastoreService, firestoreService);
}

@visibleForTesting
Expand Down
55 changes: 53 additions & 2 deletions app_dart/lib/src/service/firestore.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import 'package:github/github.dart';
import 'package:googleapis/firestore/v1.dart';
import 'package:http/http.dart';

import '../model/firestore/commit.dart';
import '../model/firestore/github_gold_status.dart';
import '../model/firestore/task.dart';
import 'access_client_provider.dart';
Expand All @@ -18,12 +19,17 @@ const String kDatabase = 'projects/${Config.flutterGcpProjectId}/databases/${Con
const String kDocumentParent = '$kDatabase/documents';
const String kFieldFilterOpEqual = 'EQUAL';
const String kCompositeFilterOpAnd = 'AND';
const String kQueryOrderDescending = 'DESCENDING';

const int kFilterStringSpaceSplitElements = 2;
const int kFilterStringSpaceSplitOpIndex = 1;

const Map<String, String> kRelationMapping = <String, String>{
'=': 'EQUAL',
'<': 'LESS_THAN',
'<=': 'LESS_THAN_OR_EQUAL',
'>': 'GREATER_THAN',
'>=': 'GREATER_THAN_OR_EQUAL',
};

class FirestoreService {
Expand Down Expand Up @@ -77,6 +83,31 @@ class FirestoreService {
return databasesDocumentsResource.commit(commitRequest, kDatabase);
}

/// Queries for recent commits.
///
/// The [limit] argument specifies the maximum number of commits to retrieve.
///
/// The returned commits will be ordered by most recent [Commit.timestamp].
Future<List<Commit>> queryRecentCommits({
int limit = 100,
int? timestamp,
String? branch,
required RepositorySlug slug,
}) async {
timestamp ??= DateTime.now().millisecondsSinceEpoch;
branch ??= Config.defaultBranch(slug);
final Map<String, Object> filterMap = <String, Object>{
'$kCommitBranchField =': branch,
'$kCommitRepositoryPathField =': slug.fullName,
'$kCommitCreateTimestampField <': timestamp,
};
final Map<String, String> orderMap = <String, String>{
kCommitCreateTimestampField: kQueryOrderDescending,
};
final List<Document> documents = await query(kCommitCollectionId, filterMap, orderMap: orderMap, limit: limit);
return documents.map((Document document) => Commit.fromDocument(commitDocument: document)).toList();
}

/// Returns all tasks running against the speificed [commitSha].
Future<List<Task>> queryCommitTasks(String commitSha) async {
final Map<String, Object> filterMap = <String, Object>{
Expand Down Expand Up @@ -161,6 +192,17 @@ class FirestoreService {
);
}

List<Order>? generateOrders(Map<String, String>? orderMap) {
if (orderMap == null || orderMap.isEmpty) {
return null;
}
final List<Order> orders = <Order>[];
orderMap.forEach((field, direction) {
orders.add(Order(field: FieldReference(fieldPath: field), direction: direction));
});
return orders;
}

/// Wrapper to simplify Firestore query.
///
/// The [filterMap] follows format:
Expand All @@ -175,15 +217,24 @@ class FirestoreService {
Future<List<Document>> query(
String collectionId,
Map<String, Object> filterMap, {
int? limit,
Map<String, String>? orderMap,
String compositeFilterOp = kCompositeFilterOpAnd,
}) async {
final ProjectsDatabasesDocumentsResource databasesDocumentsResource = await documentResource();
final List<CollectionSelector> from = <CollectionSelector>[
CollectionSelector(collectionId: collectionId),
];
final Filter filter = generateFilter(filterMap, compositeFilterOp);
final RunQueryRequest runQueryRequest =
RunQueryRequest(structuredQuery: StructuredQuery(from: from, where: filter));
final List<Order>? orders = generateOrders(orderMap);
final RunQueryRequest runQueryRequest = RunQueryRequest(
structuredQuery: StructuredQuery(
from: from,
where: filter,
orderBy: orders,
limit: limit,
),
);
final List<RunQueryResponseElement> runQueryResponseElements =
await databasesDocumentsResource.runQuery(runQueryRequest, kDocumentParent);
return documentsFromQueryResponse(runQueryResponseElements);
Expand Down
41 changes: 21 additions & 20 deletions app_dart/lib/src/service/scheduler.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import 'package:cocoon_service/src/service/exceptions.dart';
import 'package:cocoon_service/src/service/build_status_provider.dart';
import 'package:cocoon_service/src/service/scheduler/policy.dart';
import 'package:gcloud/db.dart';
import 'package:github/github.dart' as github;
import 'package:github/github.dart';
import 'package:github/hooks.dart';
import 'package:googleapis/bigquery/v2.dart';
Expand Down Expand Up @@ -64,6 +63,7 @@ class Scheduler {
final HttpClientProvider httpClientProvider;

late DatastoreService datastore;
late FirestoreService firestoreService;
LuciBuildService luciBuildService;

/// Name of the subcache to store scheduler related values in redis.
Expand All @@ -90,7 +90,7 @@ class Scheduler {
///
/// If [PullRequest] was merged, schedule prod tasks against it.
/// Otherwise if it is presubmit, schedule try tasks against it.
Future<void> addPullRequest(github.PullRequest pr) async {
Future<void> addPullRequest(PullRequest pr) async {
datastore = datastoreProvider(config.db);
// TODO(chillers): Support triggering on presubmit. https://github.com/flutter/flutter/issues/77858
if (!pr.merged!) {
Expand Down Expand Up @@ -290,7 +290,7 @@ class Scheduler {

/// Cancel all incomplete targets against a pull request.
Future<void> cancelPreSubmitTargets({
required github.PullRequest pullRequest,
required PullRequest pullRequest,
String reason = 'Newer commit available',
}) async {
await luciBuildService.cancelBuilds(pullRequest, reason);
Expand All @@ -303,7 +303,7 @@ class Scheduler {
/// Schedules a [kCiYamlCheckName] to validate [CiYaml] is valid and all builds were able to be triggered.
/// If [builderTriggerList] is specified, then trigger only those targets.
Future<void> triggerPresubmitTargets({
required github.PullRequest pullRequest,
required PullRequest pullRequest,
String reason = 'Newer commit available',
List<String>? builderTriggerList,
}) async {
Expand All @@ -315,19 +315,19 @@ class Scheduler {
);

log.info('Creating ciYaml validation check run for ${pullRequest.number}');
final github.CheckRun ciValidationCheckRun = await githubChecksService.githubChecksUtil.createCheckRun(
final CheckRun ciValidationCheckRun = await githubChecksService.githubChecksUtil.createCheckRun(
config,
pullRequest.base!.repo!.slug(),
pullRequest.head!.sha!,
kCiYamlCheckName,
output: const github.CheckRunOutput(
output: const CheckRunOutput(
title: kCiYamlCheckName,
summary: 'If this check is stuck pending, push an empty commit to retrigger the checks',
),
);

log.info('Creating presubmit targets for ${pullRequest.number}');
final github.RepositorySlug slug = pullRequest.base!.repo!.slug();
final RepositorySlug slug = pullRequest.base!.repo!.slug();
dynamic exception;
try {
// Both the author and label should be checked to make sure that no one is
Expand Down Expand Up @@ -362,8 +362,8 @@ class Scheduler {
config,
slug,
ciValidationCheckRun,
status: github.CheckRunStatus.completed,
conclusion: github.CheckRunConclusion.success,
status: CheckRunStatus.completed,
conclusion: CheckRunConclusion.success,
);
} else {
log.warning('Marking PR #${pullRequest.number} $kCiYamlCheckName as failed');
Expand All @@ -373,9 +373,9 @@ class Scheduler {
config,
slug,
ciValidationCheckRun,
status: github.CheckRunStatus.completed,
conclusion: github.CheckRunConclusion.failure,
output: github.CheckRunOutput(
status: CheckRunStatus.completed,
conclusion: CheckRunConclusion.failure,
output: CheckRunOutput(
title: kCiYamlCheckName,
summary: '.ci.yaml has failures',
text: exception.toString(),
Expand All @@ -402,20 +402,20 @@ class Scheduler {
/// 2. Get failed LUCI builds for this pull request at [commitSha].
/// 3. Rerun the failed builds that also have a failed check status.
Future<void> retryPresubmitTargets({
required github.PullRequest pullRequest,
required PullRequest pullRequest,
required CheckSuiteEvent checkSuiteEvent,
}) async {
final github.GitHub githubClient = await config.createGitHubClient(pullRequest: pullRequest);
final Map<String, github.CheckRun> checkRuns = await githubChecksService.githubChecksUtil.allCheckRuns(
final GitHub githubClient = await config.createGitHubClient(pullRequest: pullRequest);
final Map<String, CheckRun> checkRuns = await githubChecksService.githubChecksUtil.allCheckRuns(
githubClient,
checkSuiteEvent,
);
final List<Target> presubmitTargets = await getPresubmitTargets(pullRequest);
final List<Build?> failedBuilds = await luciBuildService.failedBuilds(pullRequest, presubmitTargets);
for (Build? build in failedBuilds) {
final github.CheckRun checkRun = checkRuns[build!.builderId.builder!]!;
final CheckRun checkRun = checkRuns[build!.builderId.builder!]!;

if (checkRun.status != github.CheckRunStatus.completed) {
if (checkRun.status != CheckRunStatus.completed) {
// Check run is still in progress, do not retry.
continue;
}
Expand All @@ -433,7 +433,7 @@ class Scheduler {
/// Filters targets with runIf, matching them to the diff of [pullRequest].
///
/// In the case there is an issue getting the diff from GitHub, all targets are returned.
Future<List<Target>> getPresubmitTargets(github.PullRequest pullRequest) async {
Future<List<Target>> getPresubmitTargets(PullRequest pullRequest) async {
final Commit commit = Commit(
branch: pullRequest.base!.ref,
repository: pullRequest.base!.repo!.fullName,
Expand Down Expand Up @@ -487,7 +487,7 @@ class Scheduler {
List<String> files = <String>[];
try {
files = await githubService.listFiles(pullRequest);
} on github.GitHubError catch (error) {
} on GitHubError catch (error) {
log.warning(error);
log.warning('Unable to get diff for pullRequest=$pullRequest');
log.warning('Running all targets');
Expand Down Expand Up @@ -639,7 +639,8 @@ class Scheduler {
/// to ensure new targets without `bringup: true` label are not added into the build.
Future<Commit> generateTotCommit({required String branch, required RepositorySlug slug}) async {
datastore = datastoreProvider(config.db);
final BuildStatusService buildStatusService = buildStatusProvider(datastore);
firestoreService = await config.createFirestoreService();
final BuildStatusService buildStatusService = buildStatusProvider(datastore, firestoreService);
final Commit totCommit = (await buildStatusService
.retrieveCommitStatus(
limit: 1,
Expand Down
Loading

0 comments on commit 77d6696

Please sign in to comment.