Skip to content

Commit

Permalink
Merge branch 'flutter:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@ricardoamador
ricardoamador authored Feb 21, 2024
2 parents 5b8a671 + 1c56d9e commit eb9ac86
Show file tree
Hide file tree
Showing 18 changed files with 496 additions and 123 deletions.
73 changes: 53 additions & 20 deletions app_dart/lib/src/model/firestore/task.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'package:cocoon_service/cocoon_service.dart';
import 'package:googleapis/firestore/v1.dart' hide Status;

import '../../request_handling/exceptions.dart';
Expand Down Expand Up @@ -77,24 +78,27 @@ class Task extends Document {
///
/// This is _not_ when the task first started running, as tasks start out in
/// the 'New' state until they've been picked up by an [Agent].
int? get createTimestamp => int.parse(fields!['createTimestamp']!.integerValue!);
int? get createTimestamp => int.parse(fields![kTaskCreateTimestampField]!.integerValue!);

/// The timestamp (in milliseconds since the Epoch) that this task started
/// running.
///
/// Tasks may be run more than once. If this task has been run more than
/// once, this timestamp represents when the task was most recently started.
int? get startTimestamp => int.parse(fields!['startTimestamp']!.integerValue!);
int? get startTimestamp => int.parse(fields![kTaskStartTimestampField]!.integerValue!);

/// The timestamp (in milliseconds since the Epoch) that this task last
/// finished running.
int? get endTimestamp => int.parse(fields!['endTimestamp']!.integerValue!);
int? get endTimestamp => int.parse(fields![kTaskEndTimestampField]!.integerValue!);

/// The name of the task.
///
/// This is a human-readable name, typically a test name (e.g.
/// "hello_world__memory").
String? get taskName => fields!['endTimestamp']!.stringValue!;
String? get taskName => fields![kTaskNameField]!.stringValue!;

/// The sha of the task commit.
String? get commitSha => fields![kTaskCommitShaField]!.stringValue!;

/// The number of attempts that have been made to run this task successfully.
///
Expand All @@ -109,24 +113,25 @@ class Task extends Document {
/// * <https://github.com/flutter/flutter/blob/master/.ci.yaml>
///
/// A flaky (`bringup: true`) task will not block the tree.
bool? get bringup => fields!['bringup']!.booleanValue!;
bool? get bringup => fields![kTaskBringupField]!.booleanValue!;

/// Whether the test execution of this task shows flake.
///
/// Test runner supports rerun, and this flag tracks if a flake happens.
///
/// See also:
/// * <https://github.com/flutter/flutter/blob/master/dev/devicelab/lib/framework/runner.dart>
bool? get testFlaky => fields!['testFlaky']!.booleanValue!;
bool? get testFlaky => fields![kTaskTestFlakyField]!.booleanValue!;

/// The build number of luci build: https://chromium.googlesource.com/infra/luci/luci-go/+/master/buildbucket/proto/build.proto#146
int? get buildNumber => fields!.containsKey('buildNumber') ? int.parse(fields!['buildNumber']!.integerValue!) : null;
int? get buildNumber =>
fields!.containsKey(kTaskBuildNumberField) ? int.parse(fields![kTaskBuildNumberField]!.integerValue!) : null;

/// The status of the task.
///
/// Legal values and their meanings are defined in [legalStatusValues].
String get status {
final String taskStatus = fields!['status']!.stringValue!;
final String taskStatus = fields![kTaskStatusField]!.stringValue!;
if (!legalStatusValues.contains(taskStatus)) {
throw ArgumentError('Invalid state: "$taskStatus"');
}
Expand All @@ -137,10 +142,18 @@ class Task extends Document {
if (!legalStatusValues.contains(value)) {
throw ArgumentError('Invalid state: "$value"');
}
fields!['status'] = Value(stringValue: value);
fields![kTaskStatusField] = Value(stringValue: value);
return value;
}

void setEndTimestamp(int endTimestamp) {
fields![kTaskEndTimestampField] = Value(integerValue: endTimestamp.toString());
}

void setTestFlaky(bool testFlaky) {
fields![kTaskTestFlakyField] = Value(booleanValue: testFlaky);
}

/// Update [Task] fields based on a LUCI [Build].
void updateFromBuild(Build build) {
final List<String>? tags = build.tags;
Expand All @@ -150,14 +163,34 @@ class Task extends Document {
log.warning('Tags: $tags');
throw const BadRequestException('build_address does not contain build number');
}
fields!['buildNumber'] = Value(integerValue: buildAddress.split('/').last);
fields!['createTimestamp'] = Value(integerValue: (build.createdTimestamp?.millisecondsSinceEpoch ?? 0).toString());
fields!['startTimestamp'] = Value(integerValue: (build.startedTimestamp?.millisecondsSinceEpoch ?? 0).toString());
fields!['endTimestamp'] = Value(integerValue: (build.completedTimestamp?.millisecondsSinceEpoch ?? 0).toString());
fields![kTaskBuildNumberField] = Value(integerValue: buildAddress.split('/').last);
fields![kTaskCreateTimestampField] = Value(
integerValue: (build.createdTimestamp?.millisecondsSinceEpoch ?? kTaskDefaultTimestampValue).toString(),
);
fields![kTaskStartTimestampField] = Value(
integerValue: (build.startedTimestamp?.millisecondsSinceEpoch ?? kTaskDefaultTimestampValue).toString(),
);
fields![kTaskEndTimestampField] = Value(
integerValue: (build.completedTimestamp?.millisecondsSinceEpoch ?? kTaskDefaultTimestampValue).toString(),
);

_setStatusFromLuciStatus(build);
}

void resetAsRetry({int attempt = 1}) {
name = '$kDatabase/documents/$kTaskCollectionId/${commitSha}_${taskName}_$attempt';
fields = <String, Value>{
kTaskCreateTimestampField: Value(integerValue: DateTime.now().millisecondsSinceEpoch.toString()),
kTaskEndTimestampField: Value(integerValue: kTaskDefaultTimestampValue.toString()),
kTaskBringupField: Value(booleanValue: bringup),
kTaskNameField: Value(stringValue: taskName),
kTaskStartTimestampField: Value(integerValue: kTaskDefaultTimestampValue.toString()),
kTaskStatusField: Value(stringValue: Task.statusNew),
kTaskTestFlakyField: Value(booleanValue: false),
kTaskCommitShaField: Value(stringValue: commitSha),
};
}

/// Get a [Task] status from a LUCI [Build] status/result.
String _setStatusFromLuciStatus(Build build) {
// Updates can come out of order. Ensure completed statuses are kept.
Expand Down Expand Up @@ -201,13 +234,13 @@ class Task extends Document {
String toString() {
final StringBuffer buf = StringBuffer()
..write('$runtimeType(')
..write(', createTimestamp: $createTimestamp')
..write(', startTimestamp: $startTimestamp')
..write(', endTimestamp: $endTimestamp')
..write(', name: $name')
..write(', bringup: $bringup')
..write(', testRunFlaky: $testFlaky')
..write(', status: $status')
..write(', $kTaskCreateTimestampField: $createTimestamp')
..write(', $kTaskStartTimestampField: $startTimestamp')
..write(', $kTaskEndTimestampField: $endTimestamp')
..write(', $kTaskNameField: $name')
..write(', $kTaskBringupField: $bringup')
..write(', $kTaskTestFlakyField: $testFlaky')
..write(', $kTaskStatusField: $status')
..write(')');
return buf.toString();
}
Expand Down
25 changes: 18 additions & 7 deletions app_dart/lib/src/request_handlers/postsubmit_luci_subscription.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import 'package:meta/meta.dart';

import '../model/appengine/commit.dart';
import '../model/appengine/task.dart';
import '../model/firestore/task.dart' as f;
import '../model/firestore/task.dart' as firestore;
import '../model/luci/push_message.dart';
import '../request_handling/body.dart';
import '../request_handling/exceptions.dart';
Expand Down Expand Up @@ -45,6 +45,7 @@ class PostsubmitLuciSubscription extends SubscriptionHandler {
@override
Future<Body> post() async {
final DatastoreService datastore = datastoreProvider(config.db);
final FirestoreService firestoreService = await config.createFirestoreService();

final BuildPushMessage buildPushMessage = BuildPushMessage.fromPushMessage(message);
log.fine('userData=${buildPushMessage.userData}');
Expand Down Expand Up @@ -83,12 +84,14 @@ class PostsubmitLuciSubscription extends SubscriptionHandler {
}
log.fine('Found $task');

firestore.Task taskDocument = firestore.Task();

if (_shouldUpdateTask(build, task)) {
final String oldTaskStatus = task.status;
task.updateFromBuild(build);
await datastore.insert(<Task>[task]);
try {
await updateFirestore(build, rawCommitKey, task.name!);
taskDocument = await updateFirestore(build, rawCommitKey, task.name!, firestoreService);
} catch (error) {
log.warning('Failed to update task in Firestore: $error');
}
Expand All @@ -114,6 +117,8 @@ class PostsubmitLuciSubscription extends SubscriptionHandler {
target: target,
task: task,
datastore: datastore,
taskDocument: taskDocument,
firestoreService: firestoreService,
);
log.info('Retried: $retried');
}
Expand Down Expand Up @@ -144,17 +149,23 @@ class PostsubmitLuciSubscription extends SubscriptionHandler {
}

/// Queries the task document and updates based on the latest build data.
Future<void> updateFirestore(Build build, String commitKeyId, String taskName) async {
final FirestoreService firestoreService = await config.createFirestoreService();
Future<firestore.Task> updateFirestore(
Build build,
String commitKeyId,
String taskName,
FirestoreService firestoreService,
) async {
final int currentAttempt = githubChecksService.currentAttempt(build);
final String sha = commitKeyId.split('/').last;
final String documentName = '$kDatabase/documents/tasks/${sha}_${taskName}_1';
final String documentName = '$kDatabase/documents/tasks/${sha}_${taskName}_$currentAttempt';
log.info('getting firestore document: $documentName');
final f.Task firestoreTask =
await f.Task.fromFirestore(firestoreService: firestoreService, documentName: documentName);
final firestore.Task firestoreTask =
await firestore.Task.fromFirestore(firestoreService: firestoreService, documentName: documentName);
log.info('updating firestoreTask based on build');
firestoreTask.updateFromBuild(build);
log.info('finished updating firestoreTask based on builds');
final List<Write> writes = documentsToWrites([firestoreTask], exists: true);
await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
return firestoreTask;
}
}
2 changes: 1 addition & 1 deletion app_dart/lib/src/request_handlers/presubmit_luci_subscription.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class PresubmitLuciSubscription extends SubscriptionHandler {
);
bool rescheduled = false;
if (githubChecksService.taskFailed(buildPushMessage)) {
final int currentAttempt = githubChecksService.currentAttempt(buildPushMessage);
final int currentAttempt = githubChecksService.currentAttempt(build);
final int maxAttempt = await _getMaxAttempt(buildPushMessage, slug, builderName);
if (currentAttempt < maxAttempt) {
rescheduled = true;
Expand Down
21 changes: 16 additions & 5 deletions app_dart/lib/src/request_handlers/push_gold_status_to_github.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,11 +119,22 @@ class PushGoldStatusToGithub extends ApiRequestHandler<Body> {
for (Map<String, dynamic> checkRun in checkRuns) {
log.fine('Check run: $checkRun');
final String name = checkRun['name'].toLowerCase() as String;
// Framework shards run framework goldens
// Web shards run web version of framework goldens
// Misc shard runs API docs goldens
if (name.contains('framework') || name.contains('web engine') || name.contains('misc')) {
runsGoldenFileTests = true;
if (slug == Config.engineSlug) {
if (const <String>[
'linux_android_emulator',
'linux_host_engine',
'mac_host_engine',
'linux_web_engine',
].any((String shardSubString) => name.contains(shardSubString))) {
runsGoldenFileTests = true;
}
} else if (slug == Config.flutterSlug) {
if (const <String>[
'framework',
'misc',
].any((String shardSubString) => name.contains(shardSubString))) {
runsGoldenFileTests = true;
}
}
if (checkRun['conclusion'] == null || checkRun['conclusion'].toUpperCase() != 'SUCCESS') {
incompleteChecks.add(name);
Expand Down
27 changes: 27 additions & 0 deletions app_dart/lib/src/request_handlers/update_task_status.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,17 @@
import 'dart:async';

import 'package:gcloud/db.dart';
import 'package:googleapis/firestore/v1.dart';
import 'package:meta/meta.dart';

import '../model/appengine/commit.dart';
import '../model/appengine/task.dart';
import '../model/firestore/task.dart' as firestore;
import '../request_handling/api_request_handler.dart';
import '../request_handling/body.dart';
import '../request_handling/exceptions.dart';
import '../service/datastore.dart';
import '../service/firestore.dart';
import '../service/logging.dart';

/// Endpoint for task runners to update Cocoon with test run information.
Expand Down Expand Up @@ -60,9 +63,33 @@ class UpdateTaskStatus extends ApiRequestHandler<UpdateTaskStatusResponse> {
task.isTestFlaky = isTestFlaky;

await datastore.insert(<Task>[task]);

try {
await updateTaskDocument(task.status, task.endTimestamp!, task.isTestFlaky!);
} catch (error) {
log.warning('Failed to update task in Firestore: $error');
}
return UpdateTaskStatusResponse(task);
}

Future<void> updateTaskDocument(String status, int endTimestamp, bool isTestFlaky) async {
final FirestoreService firestoreService = await config.createFirestoreService();
final String sha = (requestData![gitShaParam] as String).trim();
final String? taskName = requestData![builderNameParam] as String?;
final String documentName = '$kDatabase/documents/tasks/${sha}_${taskName}_1';
log.info('getting firestore document: $documentName');
final List<firestore.Task> initialTasks = await firestoreService.queryCommitTasks(sha);
// Targets the latest task. This assumes only one task is running at any time.
final firestore.Task firestoreTask = initialTasks.where((firestore.Task task) => task.taskName == taskName).reduce(
(firestore.Task current, firestore.Task next) => current.name!.compareTo(next.name!) > 0 ? current : next,
);
firestoreTask.setStatus(status);
firestoreTask.setEndTimestamp(endTimestamp);
firestoreTask.setTestFlaky(isTestFlaky);
final List<Write> writes = documentsToWrites([firestoreTask], exists: true);
await firestoreService.batchWriteDocuments(BatchWriteRequest(writes: writes), kDatabase);
}

/// Retrieve [Task] from [DatastoreService] when given [gitShaParam], [gitBranchParam], and [builderNameParam].
///
/// This is used when the DeviceLab test runner is uploading results to Cocoon for runs on LUCI.
Expand Down
Loading

0 comments on commit eb9ac86

Please sign in to comment.