Skip to content

Commit

Permalink
Merge branch 'flutter:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@ricardoamador
ricardoamador authored Mar 12, 2024
2 parents 724643f + d3b4fad commit caefcbf
Show file tree
Hide file tree
Showing 21 changed files with 2,538 additions and 7,877 deletions.
1 change: 1 addition & 0 deletions analyze/analyze.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ Future<void> verifyNoTrailingSpaces(
.where((File file) => path.extension(file.path) != '.ico')
.where((File file) => path.extension(file.path) != '.jar')
.where((File file) => path.extension(file.path) != '.swp')
.where((File file) => path.extension(file.path) != '.zip')
.where((File file) => !path.basename(file.path).endsWith('pbserver.dart'))
.where((File file) => !path.basename(file.path).endsWith('pb.dart'))
.where((File file) => !path.basename(file.path).endsWith('pbenum.dart'))
Expand Down
2 changes: 1 addition & 1 deletion app_dart/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


# Dart Docker official images can be found here: https://hub.docker.com/_/dart
FROM dart:beta@sha256:b166b6f9a5969fbb0feaf38480b064060f1f16eb93f7dce0b08608929f40712c
FROM dart:beta@sha256:f2d76b7a0e6e5ac35012d5a14ae587e3382d426d74b426a380f2f39f603663c8

WORKDIR /app

Expand Down
20 changes: 20 additions & 0 deletions app_dart/lib/src/model/firestore/github_gold_status.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,26 @@ class GithubGoldStatus extends Document {

int? get updates => int.parse(fields![kGithubGoldStatusUpdatesField]!.integerValue!);

String setStatus(String status) {
fields![kGithubGoldStatusStatusField] = Value(stringValue: status);
return status;
}

String setHead(String head) {
fields![kGithubGoldStatusHeadField] = Value(stringValue: head);
return head;
}

int setUpdates(int updates) {
fields![kGithubGoldStatusUpdatesField] = Value(integerValue: updates.toString());
return updates;
}

String setDescription(String description) {
fields![kGithubGoldStatusDescriptionField] = Value(stringValue: description);
return description;
}

/// A serializable form of [slug].
///
/// This will be of the form `<org>/<repo>`. e.g. `flutter/flutter`.
Expand Down
57 changes: 30 additions & 27 deletions app_dart/lib/src/request_handlers/push_gold_status_to_github.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,35 +39,39 @@ class PushGoldStatusToGithub extends ApiRequestHandler<Body> {
@override
Future<Body> get() async {
final DatastoreService datastore = datastoreProvider(config.db);
final FirestoreService firestoreService = await config.createFirestoreService();

if (authContext!.clientContext.isDevelopmentEnvironment) {
// Don't push gold status from the local dev server.
return Body.empty;
}

await _sendStatusUpdates(datastore, Config.flutterSlug);
await _sendStatusUpdates(datastore, Config.engineSlug);
await _sendStatusUpdates(datastore, firestoreService, Config.flutterSlug);
await _sendStatusUpdates(datastore, firestoreService, Config.engineSlug);

return Body.empty;
}

Future<void> _sendStatusUpdates(
DatastoreService datastore,
FirestoreService firestoreService,
RepositorySlug slug,
) async {
final GitHub gitHubClient = await config.createGitHubClient(slug: slug);
final List<GithubGoldStatusUpdate> statusUpdates = <GithubGoldStatusUpdate>[];
final List<GithubGoldStatus> githubGoldStatuses = <GithubGoldStatus>[];
log.fine('Beginning Gold checks...');
await for (PullRequest pr in gitHubClient.pullRequests.list(slug)) {
assert(pr.number != null);
// Get last known Gold status from datastore.
// Get last known Gold status from firestore.
final GithubGoldStatusUpdate lastUpdate = await datastore.queryLastGoldUpdate(slug, pr);
final GithubGoldStatus githubGoldStatus = await firestoreService.queryLastGoldStatus(slug, pr.number!);
CreateStatus statusRequest;

log.fine('Last known Gold status for $slug#${pr.number} was with sha: '
'${lastUpdate.head}, status: ${lastUpdate.status}, description: ${lastUpdate.description}');
'${githubGoldStatus.head}, status: ${githubGoldStatus.status}, description: ${githubGoldStatus.description}');

if (lastUpdate.status == GithubGoldStatusUpdate.statusCompleted && lastUpdate.head == pr.head!.sha) {
if (githubGoldStatus.status == GithubGoldStatus.statusCompleted && githubGoldStatus.head == pr.head!.sha) {
log.fine('Completed status already reported for this commit.');
// We have already seen this commit and it is completed or, this is not
// a change staged to land on master, which we should ignore.
Expand All @@ -90,16 +94,16 @@ class PushGoldStatusToGithub extends ApiRequestHandler<Body> {
// been a new commit, we cannot rescind a previously posted status, so
// if it is already pending, we should make the contributor aware of
// that fact.
if (lastUpdate.status == GithubGoldStatusUpdate.statusRunning &&
lastUpdate.head == pr.head!.sha &&
if (githubGoldStatus.status == GithubGoldStatus.statusRunning &&
githubGoldStatus.head == pr.head!.sha &&
!await _alreadyCommented(gitHubClient, pr, slug, config.flutterGoldDraftChange)) {
await gitHubClient.issues
.createComment(slug, pr.number!, config.flutterGoldDraftChange + config.flutterGoldAlertConstant(slug));
}
continue;
}

log.fine('Querying builds for pull request #${pr.number} with sha: ${lastUpdate.head}...');
log.fine('Querying builds for pull request #${pr.number} with sha: ${githubGoldStatus.head}...');
final GraphQLClient gitHubGraphQLClient = await config.createGitHubGraphQLClient();
final List<String> incompleteChecks = <String>[];
bool runsGoldenFileTests = false;
Expand Down Expand Up @@ -164,21 +168,20 @@ class PushGoldStatusToGithub extends ApiRequestHandler<Body> {
// should just be pending. Any draft PRs are skipped
// until marked ready for review.
log.fine('Waiting for checks to be completed.');
statusRequest =
_createStatus(GithubGoldStatusUpdate.statusRunning, config.flutterGoldPending, slug, pr.number!);
statusRequest = _createStatus(GithubGoldStatus.statusRunning, config.flutterGoldPending, slug, pr.number!);
} else {
// We do not want to query Gold on a draft PR.
assert(!pr.draft!);
// Get Gold status.
final String goldStatus = await _getGoldStatus(slug, pr);
statusRequest = _createStatus(
goldStatus,
goldStatus == GithubGoldStatusUpdate.statusRunning ? config.flutterGoldChanges : config.flutterGoldSuccess,
goldStatus == GithubGoldStatus.statusRunning ? config.flutterGoldChanges : config.flutterGoldSuccess,
slug,
pr.number!,
);
log.fine('New status for potential update: ${statusRequest.state}, ${statusRequest.description}');
if (goldStatus == GithubGoldStatusUpdate.statusRunning &&
if (goldStatus == GithubGoldStatus.statusRunning &&
!await _alreadyCommented(gitHubClient, pr, slug, config.flutterGoldCommentID(pr))) {
log.fine('Notifying for triage.');
await _commentAndApplyGoldLabels(gitHubClient, pr, slug);
Expand All @@ -187,7 +190,7 @@ class PushGoldStatusToGithub extends ApiRequestHandler<Body> {

// Push updates if there is a status change (detected by unique description)
// or this is a new commit.
if (lastUpdate.description != statusRequest.description || lastUpdate.head != pr.head!.sha) {
if (githubGoldStatus.description != statusRequest.description || githubGoldStatus.head != pr.head!.sha) {
try {
log.fine('Pushing status to GitHub: ${statusRequest.state}, ${statusRequest.description}');
await gitHubClient.repositories.createStatus(slug, pr.head!.sha!, statusRequest);
Expand All @@ -196,6 +199,12 @@ class PushGoldStatusToGithub extends ApiRequestHandler<Body> {
lastUpdate.updates = (lastUpdate.updates ?? 0) + 1;
lastUpdate.description = statusRequest.description!;
statusUpdates.add(lastUpdate);

githubGoldStatus.setStatus(statusRequest.state!);
githubGoldStatus.setHead(pr.head!.sha!);
githubGoldStatus.setUpdates((githubGoldStatus.updates ?? 0) + 1);
githubGoldStatus.setDescription(statusRequest.description!);
githubGoldStatuses.add(githubGoldStatus);
} catch (error) {
log.severe('Failed to post status update to ${slug.fullName}#${pr.number}: $error');
}
Expand All @@ -206,24 +215,18 @@ class PushGoldStatusToGithub extends ApiRequestHandler<Body> {
}
await datastore.insert(statusUpdates);
log.fine('Committed all updates for $slug');

// TODO(keyonghan): remove try block after fully migrated to firestore
// https://github.com/flutter/flutter/issues/142951
try {
await updateGithubGoldStatusDocuments(statusUpdates);
} catch (error) {
log.warning('Failed to update github gold status in Firestore: $error');
}
await updateGithubGoldStatusDocuments(githubGoldStatuses, firestoreService);
log.fine('Saved all updates to firestore for $slug');
}

Future<void> updateGithubGoldStatusDocuments(List<GithubGoldStatusUpdate> statusUpdates) async {
if (statusUpdates.isEmpty) {
Future<void> updateGithubGoldStatusDocuments(
List<GithubGoldStatus> githubGoldStatuses,
FirestoreService firestoreService,
) async {
if (githubGoldStatuses.isEmpty) {
return;
}
final List<GithubGoldStatus> githubGoldStatusDocuments =
statusUpdates.map((e) => githubGoldStatusToDocument(e)).toList();
final List<Write> writes = documentsToWrites(githubGoldStatusDocuments);
final FirestoreService firestoreService = await config.createFirestoreService();
final List<Write> writes = documentsToWrites(githubGoldStatuses);
await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
}

Expand Down
137 changes: 125 additions & 12 deletions app_dart/lib/src/service/firestore.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,26 @@
import 'dart:async';

import 'package:cocoon_service/cocoon_service.dart';
import 'package:github/github.dart';
import 'package:googleapis/firestore/v1.dart';
import 'package:http/http.dart';

import '../model/firestore/task.dart' as firestore;
import '../model/firestore/github_gold_status.dart';
import '../model/firestore/task.dart';
import 'access_client_provider.dart';
import 'config.dart';

const String kDatabase = 'projects/${Config.flutterGcpProjectId}/databases/${Config.flutterGcpFirestoreDatabase}';
const String kDocumentParent = '$kDatabase/documents';
const String kFieldFilterOpEqual = 'EQUAL';
const String kCompositeFilterOpAnd = 'AND';

const int kFilterStringSpaceSplitElements = 2;
const int kFilterStringSpaceSplitOpIndex = 1;

const Map<String, String> kRelationMapping = <String, String>{
'=': 'EQUAL',
};

class FirestoreService {
const FirestoreService(this.accessClientProvider);
Expand Down Expand Up @@ -67,24 +77,127 @@ class FirestoreService {
return databasesDocumentsResource.commit(commitRequest, kDatabase);
}

Future<List<firestore.Task>> queryCommitTasks(String commitSha) async {
/// Returns all tasks running against the speificed [commitSha].
Future<List<Task>> queryCommitTasks(String commitSha) async {
final Map<String, Object> filterMap = <String, Object>{
'$kTaskCommitShaField =': commitSha,
};
final List<Document> documents = await query(kTaskCollectionId, filterMap);
return documents.map((Document document) => Task.fromDocument(taskDocument: document)).toList();
}

/// Queries the last updated Gold status for the [slug] and [prNumber].
///
/// If not existing, returns a fresh new Gold status.
Future<GithubGoldStatus> queryLastGoldStatus(RepositorySlug slug, int prNumber) async {
final Map<String, Object> filterMap = <String, Object>{
'$kGithubGoldStatusPrNumberField =': prNumber,
'$kGithubGoldStatusRepositoryField =': slug.fullName,
};
final List<Document> documents = await query(kGithubGoldStatusCollectionId, filterMap);
final List<GithubGoldStatus> githubGoldStatuses =
documents.map((Document document) => GithubGoldStatus.fromDocument(githubGoldStatus: document)).toList();
if (githubGoldStatuses.isEmpty) {
return GithubGoldStatus.fromDocument(
githubGoldStatus: Document(
name: '$kDatabase/documents/$kGithubGoldStatusCollectionId/${slug.owner}_${slug.name}_$prNumber',
fields: <String, Value>{
kGithubGoldStatusPrNumberField: Value(integerValue: prNumber.toString()),
kGithubGoldStatusHeadField: Value(stringValue: ''),
kGithubGoldStatusStatusField: Value(stringValue: ''),
kGithubGoldStatusUpdatesField: Value(integerValue: '0'),
kGithubGoldStatusDescriptionField: Value(stringValue: ''),
kGithubGoldStatusRepositoryField: Value(stringValue: slug.fullName),
},
),
);
} else {
if (githubGoldStatuses.length > 1) {
throw StateError('GithubGoldStatusUpdate should have no more than one entry on '
'repository ${slug.fullName}, pr $prNumber.');
}
return githubGoldStatuses.single;
}
}

/// Returns Firestore [Value] based on corresponding object type.
Value getValueFromFilter(Object comparisonOject) {
if (comparisonOject is int) {
return Value(integerValue: comparisonOject.toString());
} else if (comparisonOject is bool) {
return Value(booleanValue: comparisonOject);
}
return Value(stringValue: comparisonOject as String);
}

/// Generates Firestore query filter based on "human" read conditions.
Filter generateFilter(Map<String, Object> filterMap, String compositeFilterOp) {
final List<Filter> filters = <Filter>[];
filterMap.forEach((filterString, comparisonOject) {
final List<String> parts = filterString.split(' ');
if (parts.length != kFilterStringSpaceSplitElements ||
!kRelationMapping.containsKey(parts[kFilterStringSpaceSplitOpIndex])) {
throw ArgumentError("Invalid filter string '$filterString'.");
}
final String name = parts[0];
final String comparison = kRelationMapping[parts[1]]!;
final Value value = getValueFromFilter(comparisonOject);

filters.add(
Filter(
fieldFilter: FieldFilter(
field: FieldReference(fieldPath: name),
op: comparison,
value: value,
),
),
);
});
return Filter(
compositeFilter: CompositeFilter(
filters: filters,
op: compositeFilterOp,
),
);
}

/// Wrapper to simplify Firestore query.
///
/// The [filterMap] follows format:
/// {
/// 'fieldInt =': 1,
/// 'fieldString =': 'string',
/// 'fieldBool =': true,
/// }
/// Note
/// 1. the space in the key, which will be used to retrieve the field name and operator.
/// 2. the value could be any type, like int, string, bool, etc.
Future<List<Document>> query(
String collectionId,
Map<String, Object> filterMap, {
String compositeFilterOp = kCompositeFilterOpAnd,
}) async {
final ProjectsDatabasesDocumentsResource databasesDocumentsResource = await documentResource();
final List<CollectionSelector> from = <CollectionSelector>[
CollectionSelector(collectionId: firestore.kTaskCollectionId),
CollectionSelector(collectionId: collectionId),
];
final Filter filter = Filter(
fieldFilter: FieldFilter(
field: FieldReference(fieldPath: firestore.kTaskCommitShaField),
op: kFieldFilterOpEqual,
value: Value(stringValue: commitSha),
),
);
final Filter filter = generateFilter(filterMap, compositeFilterOp);
final RunQueryRequest runQueryRequest =
RunQueryRequest(structuredQuery: StructuredQuery(from: from, where: filter));
final List<RunQueryResponseElement> runQueryResponseElements =
await databasesDocumentsResource.runQuery(runQueryRequest, kDocumentParent);
final List<Document> documents = runQueryResponseElements.map((e) => e.document!).toList();
return documents.map((Document document) => firestore.Task.fromDocument(taskDocument: document)).toList();
return documentsFromQueryResponse(runQueryResponseElements);
}

/// Retrieve documents based on query response.
List<Document> documentsFromQueryResponse(List<RunQueryResponseElement> runQueryResponseElements) {
final List<Document> documents = <Document>[];
for (RunQueryResponseElement runQueryResponseElement in runQueryResponseElements) {
if (runQueryResponseElement.document != null) {
documents.add(runQueryResponseElement.document!);
}
}
return documents;
}
}

Expand Down
2 changes: 1 addition & 1 deletion app_dart/pubspec.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ dependencies:
file: 7.0.0
fixnum: 1.1.0
gcloud: 0.8.12
github: 9.23.0
github: 9.24.0
googleapis: 12.0.0
googleapis_auth: 1.5.0
gql: 1.0.1-alpha+1696717343881
Expand Down
Loading

0 comments on commit caefcbf

Please sign in to comment.