Merge branch 'flutter:main' into main

ricardoamador · Mar 6, 2024 · 724643f · 724643f
2 parents f4710dd + 06af2aa
commit 724643f
Show file tree

Hide file tree

Showing 52 changed files with 816 additions and 244 deletions.
diff --git a/app_dart/lib/src/model/firestore/commit.dart b/app_dart/lib/src/model/firestore/commit.dart
@@ -7,6 +7,16 @@ import 'package:github/github.dart';
 import 'package:googleapis/firestore/v1.dart' hide Status;
 
 import '../../service/firestore.dart';
+import '../appengine/commit.dart' as datastore;
+
+const String kCommitCollectionId = 'commits';
+const String kCommitAvatarField = 'avatar';
+const String kCommitBranchField = 'branch';
+const String kCommitCreateTimestampField = 'createTimestamp';
+const String kCommitAuthorField = 'author';
+const String kCommitMessageField = 'message';
+const String kCommitRepositoryPathField = 'repositoryPath';
+const String kCommitShaField = 'sha';
 
 class Commit extends Document {
   /// Lookup [Commit] from Firestore.
@@ -76,3 +86,21 @@ class Commit extends Document {
     return buf.toString();
   }
 }
+
+/// Generates commit document based on datastore commit data model.
+Commit commitToCommitDocument(datastore.Commit commit) {
+  return Commit.fromDocument(
+    commitDocument: Document(
+      name: '$kDatabase/documents/$kCommitCollectionId/${commit.sha}',
+      fields: <String, Value>{
+        kCommitAvatarField: Value(stringValue: commit.authorAvatarUrl),
+        kCommitBranchField: Value(stringValue: commit.branch),
+        kCommitCreateTimestampField: Value(integerValue: commit.timestamp.toString()),
+        kCommitAuthorField: Value(stringValue: commit.author),
+        kCommitMessageField: Value(stringValue: commit.message),
+        kCommitRepositoryPathField: Value(stringValue: commit.repository),
+        kCommitShaField: Value(stringValue: commit.sha),
+      },
+    ),
+  );
+}
diff --git a/app_dart/lib/src/model/firestore/github_gold_status.dart b/app_dart/lib/src/model/firestore/github_gold_status.dart
@@ -7,6 +7,15 @@ import 'package:github/github.dart';
 import 'package:googleapis/firestore/v1.dart' hide Status;
 
 import '../../service/firestore.dart';
+import '../appengine/github_gold_status_update.dart';
+
+const String kGithubGoldStatusCollectionId = 'githubGoldStatuses';
+const String kGithubGoldStatusPrNumberField = 'prNumber';
+const String kGithubGoldStatusHeadField = 'head';
+const String kGithubGoldStatusStatusField = 'status';
+const String kGithubGoldStatusDescriptionField = 'description';
+const String kGithubGoldStatusUpdatesField = 'updates';
+const String kGithubGoldStatusRepositoryField = 'repository';
 
 class GithubGoldStatus extends Document {
   /// Lookup [GithubGoldStatus] from Firestore.
@@ -71,3 +80,22 @@ class GithubGoldStatus extends Document {
     return buf.toString();
   }
 }
+
+/// Generates GithubGoldStatus document based on datastore GithubGoldStatusUpdate data model.
+GithubGoldStatus githubGoldStatusToDocument(GithubGoldStatusUpdate githubGoldStatus) {
+  // Prefers `_` instead of `/` in Firestore document names.
+  final String repo = githubGoldStatus.repository!.replaceAll('/', '_');
+  return GithubGoldStatus.fromDocument(
+    githubGoldStatus: Document(
+      name: '$kDatabase/documents/$kGithubGoldStatusCollectionId/${repo}_${githubGoldStatus.pr}',
+      fields: <String, Value>{
+        kGithubGoldStatusDescriptionField: Value(stringValue: githubGoldStatus.description),
+        kGithubGoldStatusHeadField: Value(stringValue: githubGoldStatus.head),
+        kGithubGoldStatusPrNumberField: Value(integerValue: githubGoldStatus.pr.toString()),
+        kGithubGoldStatusRepositoryField: Value(stringValue: githubGoldStatus.repository),
+        kGithubGoldStatusStatusField: Value(stringValue: githubGoldStatus.status),
+        kGithubGoldStatusUpdatesField: Value(integerValue: githubGoldStatus.updates.toString()),
+      },
+    ),
+  );
+}
diff --git a/app_dart/lib/src/model/firestore/task.dart b/app_dart/lib/src/model/firestore/task.dart
@@ -8,8 +8,24 @@ import 'package:googleapis/firestore/v1.dart' hide Status;
 import '../../request_handling/exceptions.dart';
 import '../../service/firestore.dart';
 import '../../service/logging.dart';
+import '../appengine/commit.dart';
+import '../appengine/task.dart' as datastore;
+import '../ci_yaml/target.dart';
 import '../luci/push_message.dart';
 
+const String kTaskCollectionId = 'tasks';
+const int kTaskDefaultTimestampValue = 0;
+const int kTaskInitialAttempt = 1;
+const String kTaskBringupField = 'bringup';
+const String kTaskBuildNumberField = 'buildNumber';
+const String kTaskCommitShaField = 'commitSha';
+const String kTaskCreateTimestampField = 'createTimestamp';
+const String kTaskEndTimestampField = 'endTimestamp';
+const String kTaskNameField = 'name';
+const String kTaskStartTimestampField = 'startTimestamp';
+const String kTaskStatusField = 'status';
+const String kTaskTestFlakyField = 'testFlaky';
+
 class Task extends Document {
   /// Lookup [Task] from Firestore.
   ///
@@ -245,3 +261,45 @@ class Task extends Document {
     return buf.toString();
   }
 }
+
+/// Generates task documents based on targets.
+List<Task> targetsToTaskDocuments(Commit commit, List<Target> targets) {
+  final Iterable<Task> iterableDocuments = targets.map(
+    (Target target) => Task.fromDocument(
+      taskDocument: Document(
+        name: '$kDatabase/documents/$kTaskCollectionId/${commit.sha}_${target.value.name}_$kTaskInitialAttempt',
+        fields: <String, Value>{
+          kTaskCreateTimestampField: Value(integerValue: commit.timestamp!.toString()),
+          kTaskEndTimestampField: Value(integerValue: kTaskDefaultTimestampValue.toString()),
+          kTaskBringupField: Value(booleanValue: target.value.bringup),
+          kTaskNameField: Value(stringValue: target.value.name),
+          kTaskStartTimestampField: Value(integerValue: kTaskDefaultTimestampValue.toString()),
+          kTaskStatusField: Value(stringValue: Task.statusNew),
+          kTaskTestFlakyField: Value(booleanValue: false),
+          kTaskCommitShaField: Value(stringValue: commit.sha),
+        },
+      ),
+    ),
+  );
+  return iterableDocuments.toList();
+}
+
+/// Generates task document based on datastore task data model.
+Task taskToDocument(datastore.Task task) {
+  final String commitSha = task.commitKey!.id!.split('/').last;
+  return Task.fromDocument(
+    taskDocument: Document(
+      name: '$kDatabase/documents/$kTaskCollectionId/${commitSha}_${task.name}_${task.attempts}',
+      fields: <String, Value>{
+        kTaskCreateTimestampField: Value(integerValue: task.createTimestamp.toString()),
+        kTaskEndTimestampField: Value(integerValue: task.endTimestamp.toString()),
+        kTaskBringupField: Value(booleanValue: task.isFlaky),
+        kTaskNameField: Value(stringValue: task.name),
+        kTaskStartTimestampField: Value(integerValue: task.startTimestamp.toString()),
+        kTaskStatusField: Value(stringValue: task.status),
+        kTaskTestFlakyField: Value(booleanValue: task.isTestFlaky),
+        kTaskCommitShaField: Value(stringValue: commitSha),
+      },
+    ),
+  );
+}
diff --git a/app_dart/lib/src/request_handlers/dart_internal_subscription.dart b/app_dart/lib/src/request_handlers/dart_internal_subscription.dart
@@ -104,7 +104,7 @@ class DartInternalSubscription extends SubscriptionHandler {
     await datastore.insert(<Task>[taskToInsert]);
     try {
       final FirestoreService firestoreService = await config.createFirestoreService();
-      final firestore.Task taskDocument = taskToTaskDocument(taskToInsert);
+      final firestore.Task taskDocument = firestore.taskToDocument(taskToInsert);
       final List<Write> writes = documentsToWrites([taskDocument]);
       await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
     } catch (error) {

diff --git a/app_dart/lib/src/request_handlers/reset_prod_task.dart b/app_dart/lib/src/request_handlers/reset_prod_task.dart
@@ -146,7 +146,8 @@ class ResetProdTask extends ApiRequestHandler<Body> {
 
     firestore.Task? taskDocument;
     final int currentAttempt = task.attempts!;
-    final String documentName = '$kDatabase/documents/$kTaskCollectionId/${sha}_${taskName}_$currentAttempt';
+    final String documentName =
+        '$kDatabase/documents/${firestore.kTaskCollectionId}/${sha}_${taskName}_$currentAttempt';
     try {
       taskDocument = await firestore.Task.fromFirestore(firestoreService: firestoreService, documentName: documentName);
     } catch (error) {

diff --git a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart
@@ -120,7 +120,7 @@ class BatchBackfiller extends RequestHandler {
     if (tasks.isEmpty) {
       return;
     }
-    final List<firestore.Task> taskDocuments = tasks.map((e) => taskToTaskDocument(e)).toList();
+    final List<firestore.Task> taskDocuments = tasks.map((e) => firestore.taskToDocument(e)).toList();
     final List<Write> writes = documentsToWrites(taskDocuments, exists: true);
     final FirestoreService firestoreService = await config.createFirestoreService();
     await firestoreService.writeViaTransaction(writes);

diff --git a/app_dart/lib/src/request_handlers/scheduler/vacuum_stale_tasks.dart b/app_dart/lib/src/request_handlers/scheduler/vacuum_stale_tasks.dart
@@ -74,7 +74,7 @@ class VacuumStaleTasks extends RequestHandler<Body> {
     if (tasks.isEmpty) {
       return;
     }
-    final List<firestore.Task> taskDocuments = tasks.map((e) => taskToTaskDocument(e)).toList();
+    final List<firestore.Task> taskDocuments = tasks.map((e) => firestore.taskToDocument(e)).toList();
     final List<Write> writes = documentsToWrites(taskDocuments, exists: true);
     final FirestoreService firestoreService = await config.createFirestoreService();
     await firestoreService.writeViaTransaction(writes);

diff --git a/app_dart/lib/src/service/commit_service.dart b/app_dart/lib/src/service/commit_service.dart
@@ -94,7 +94,7 @@ class CommitService {
       log.info('Commit does not exist in datastore, inserting into datastore');
       await datastore.insert(<Commit>[commit]);
       try {
-        final firestore.Commit commitDocument = commitToCommitDocument(commit);
+        final firestore.Commit commitDocument = firestore.commitToCommitDocument(commit);
         final List<Write> writes = documentsToWrites([commitDocument], exists: false);
         await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
       } catch (error) {

diff --git a/app_dart/lib/src/service/firestore.dart b/app_dart/lib/src/service/firestore.dart
@@ -8,50 +8,14 @@ import 'package:cocoon_service/cocoon_service.dart';
 import 'package:googleapis/firestore/v1.dart';
 import 'package:http/http.dart';
 
-import '../model/appengine/commit.dart';
-import '../model/appengine/github_gold_status_update.dart';
-import '../model/appengine/task.dart';
-import '../model/firestore/commit.dart' as firestore_comit;
-import '../model/firestore/github_gold_status.dart';
 import '../model/firestore/task.dart' as firestore;
-import '../model/ci_yaml/target.dart';
 import 'access_client_provider.dart';
 import 'config.dart';
 
 const String kDatabase = 'projects/${Config.flutterGcpProjectId}/databases/${Config.flutterGcpFirestoreDatabase}';
 const String kDocumentParent = '$kDatabase/documents';
 const String kFieldFilterOpEqual = 'EQUAL';
 
-const String kTaskCollectionId = 'tasks';
-const int kTaskDefaultTimestampValue = 0;
-const int kTaskInitialAttempt = 1;
-const String kTaskBringupField = 'bringup';
-const String kTaskBuildNumberField = 'buildNumber';
-const String kTaskCommitShaField = 'commitSha';
-const String kTaskCreateTimestampField = 'createTimestamp';
-const String kTaskEndTimestampField = 'endTimestamp';
-const String kTaskNameField = 'name';
-const String kTaskStartTimestampField = 'startTimestamp';
-const String kTaskStatusField = 'status';
-const String kTaskTestFlakyField = 'testFlaky';
-
-const String kCommitCollectionId = 'commits';
-const String kCommitAvatarField = 'avatar';
-const String kCommitBranchField = 'branch';
-const String kCommitCreateTimestampField = 'createTimestamp';
-const String kCommitAuthorField = 'author';
-const String kCommitMessageField = 'message';
-const String kCommitRepositoryPathField = 'repositoryPath';
-const String kCommitShaField = 'sha';
-
-const String kGithubGoldStatusCollectionId = 'githubGoldStatuses';
-const String kGithubGoldStatusPrNumberField = 'prNumber';
-const String kGithubGoldStatusHeadField = 'head';
-const String kGithubGoldStatusStatusField = 'status';
-const String kGithubGoldStatusDescriptionField = 'description';
-const String kGithubGoldStatusUpdatesField = 'updates';
-const String kGithubGoldStatusRepositoryField = 'repository';
-
 class FirestoreService {
   const FirestoreService(this.accessClientProvider);
 
@@ -105,10 +69,12 @@ class FirestoreService {
 
   Future<List<firestore.Task>> queryCommitTasks(String commitSha) async {
     final ProjectsDatabasesDocumentsResource databasesDocumentsResource = await documentResource();
-    final List<CollectionSelector> from = <CollectionSelector>[CollectionSelector(collectionId: kTaskCollectionId)];
+    final List<CollectionSelector> from = <CollectionSelector>[
+      CollectionSelector(collectionId: firestore.kTaskCollectionId),
+    ];
     final Filter filter = Filter(
       fieldFilter: FieldFilter(
-        field: FieldReference(fieldPath: kTaskCommitShaField),
+        field: FieldReference(fieldPath: firestore.kTaskCommitShaField),
         op: kFieldFilterOpEqual,
         value: Value(stringValue: commitSha),
       ),
@@ -122,83 +88,6 @@ class FirestoreService {
   }
 }
 
-/// Generates task documents based on targets.
-List<firestore.Task> targetsToTaskDocuments(Commit commit, List<Target> targets) {
-  final Iterable<firestore.Task> iterableDocuments = targets.map(
-    (Target target) => firestore.Task.fromDocument(
-      taskDocument: Document(
-        name: '$kDatabase/documents/$kTaskCollectionId/${commit.sha}_${target.value.name}_$kTaskInitialAttempt',
-        fields: <String, Value>{
-          kTaskCreateTimestampField: Value(integerValue: commit.timestamp!.toString()),
-          kTaskEndTimestampField: Value(integerValue: kTaskDefaultTimestampValue.toString()),
-          kTaskBringupField: Value(booleanValue: target.value.bringup),
-          kTaskNameField: Value(stringValue: target.value.name),
-          kTaskStartTimestampField: Value(integerValue: kTaskDefaultTimestampValue.toString()),
-          kTaskStatusField: Value(stringValue: Task.statusNew),
-          kTaskTestFlakyField: Value(booleanValue: false),
-          kTaskCommitShaField: Value(stringValue: commit.sha),
-        },
-      ),
-    ),
-  );
-  return iterableDocuments.toList();
-}
-
-/// Generates commit document based on datastore commit data model.
-firestore_comit.Commit commitToCommitDocument(Commit commit) {
-  return firestore_comit.Commit.fromDocument(
-    commitDocument: Document(
-      name: '$kDatabase/documents/$kCommitCollectionId/${commit.sha}',
-      fields: <String, Value>{
-        kCommitAvatarField: Value(stringValue: commit.authorAvatarUrl),
-        kCommitBranchField: Value(stringValue: commit.branch),
-        kCommitCreateTimestampField: Value(integerValue: commit.timestamp.toString()),
-        kCommitAuthorField: Value(stringValue: commit.author),
-        kCommitMessageField: Value(stringValue: commit.message),
-        kCommitRepositoryPathField: Value(stringValue: commit.repository),
-        kCommitShaField: Value(stringValue: commit.sha),
-      },
-    ),
-  );
-}
-
-/// Generates task document based on datastore task data model.
-firestore.Task taskToTaskDocument(Task task) {
-  final String commitSha = task.commitKey!.id!.split('/').last;
-  return firestore.Task.fromDocument(
-    taskDocument: Document(
-      name: '$kDatabase/documents/$kTaskCollectionId/${commitSha}_${task.name}_${task.attempts}',
-      fields: <String, Value>{
-        kTaskCreateTimestampField: Value(integerValue: task.createTimestamp.toString()),
-        kTaskEndTimestampField: Value(integerValue: task.endTimestamp.toString()),
-        kTaskBringupField: Value(booleanValue: task.isFlaky),
-        kTaskNameField: Value(stringValue: task.name),
-        kTaskStartTimestampField: Value(integerValue: task.startTimestamp.toString()),
-        kTaskStatusField: Value(stringValue: task.status),
-        kTaskTestFlakyField: Value(booleanValue: task.isTestFlaky),
-        kTaskCommitShaField: Value(stringValue: commitSha),
-      },
-    ),
-  );
-}
-
-/// Generates GithubGoldStatus document based on datastore GithubGoldStatusUpdate data model.
-GithubGoldStatus githubGoldStatusToDocument(GithubGoldStatusUpdate githubGoldStatus) {
-  return GithubGoldStatus.fromDocument(
-    githubGoldStatus: Document(
-      name: '$kDatabase/documents/$kGithubGoldStatusCollectionId/${githubGoldStatus.head}_${githubGoldStatus.pr}',
-      fields: <String, Value>{
-        kGithubGoldStatusDescriptionField: Value(stringValue: githubGoldStatus.description),
-        kGithubGoldStatusHeadField: Value(stringValue: githubGoldStatus.head),
-        kGithubGoldStatusPrNumberField: Value(integerValue: githubGoldStatus.pr.toString()),
-        kGithubGoldStatusRepositoryField: Value(stringValue: githubGoldStatus.repository),
-        kGithubGoldStatusStatusField: Value(stringValue: githubGoldStatus.status),
-        kGithubGoldStatusUpdatesField: Value(integerValue: githubGoldStatus.updates.toString()),
-      },
-    ),
-  );
-}
-
 /// Creates a list of [Write] based on documents.
 ///
 /// Null `exists` means either update when a document exists or insert when a document doesn't.

diff --git a/app_dart/lib/src/service/scheduler.dart b/app_dart/lib/src/service/scheduler.dart
@@ -167,8 +167,8 @@ class Scheduler {
 
     await _batchScheduleBuilds(commit, toBeScheduled);
     await _uploadToBigQuery(commit);
-    final firestore_commmit.Commit commitDocument = commitToCommitDocument(commit);
-    final List<firestore.Task> taskDocuments = targetsToTaskDocuments(commit, initialTargets);
+    final firestore_commmit.Commit commitDocument = firestore_commmit.commitToCommitDocument(commit);
+    final List<firestore.Task> taskDocuments = firestore.targetsToTaskDocuments(commit, initialTargets);
     final List<Write> writes = documentsToWrites([...taskDocuments, commitDocument], exists: false);
     final FirestoreService firestoreService = await config.createFirestoreService();
     // TODO(keyonghan): remove try catch logic after validated to work.

diff --git a/app_dart/test/model/firestore/commit_test.dart b/app_dart/test/model/firestore/commit_test.dart
@@ -2,7 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'package:cocoon_service/src/model/appengine/commit.dart' as datastore;
 import 'package:cocoon_service/src/model/firestore/commit.dart';
+import 'package:cocoon_service/src/service/firestore.dart';
 import 'package:mockito/mockito.dart';
 import 'package:test/test.dart';
 
@@ -36,4 +38,17 @@ void main() {
       expect(resultedCommit.fields, commit.fields);
     });
   });
+
+  test('creates commit document correctly from commit data model', () async {
+    final datastore.Commit commit = generateCommit(1);
+    final Commit commitDocument = commitToCommitDocument(commit);
+    expect(commitDocument.name, '$kDatabase/documents/$kCommitCollectionId/${commit.sha}');
+    expect(commitDocument.fields![kCommitAvatarField]!.stringValue, commit.authorAvatarUrl);
+    expect(commitDocument.fields![kCommitBranchField]!.stringValue, commit.branch);
+    expect(commitDocument.fields![kCommitCreateTimestampField]!.integerValue, commit.timestamp.toString());
+    expect(commitDocument.fields![kCommitAuthorField]!.stringValue, commit.author);
+    expect(commitDocument.fields![kCommitMessageField]!.stringValue, commit.message);
+    expect(commitDocument.fields![kCommitRepositoryPathField]!.stringValue, commit.repository);
+    expect(commitDocument.fields![kCommitShaField]!.stringValue, commit.sha);
+  });
 }