Skip to content

Commit

Permalink
Add logics to fetch commit status from Firestore (flutter#3634)
Browse files Browse the repository at this point in the history 
This is part of flutter/flutter#142951

This PR adds logics to fetch commit tasks status from Firestore based on newly added Firestore data model protos. After this lands, next step is to switch the build logic to reference this PR's changes from Datastore.

This PR is a no-op change by this point.
  • Loading branch information
@keyonghan
keyonghan authored Apr 11, 2024
1 parent 9ccd1e7 commit bbfd8e8
Show file tree
Hide file tree
Showing 5 changed files with 248 additions and 1 deletion.
123 changes: 122 additions & 1 deletion dashboard/lib/service/appengine_cocoon.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ import 'package:http/http.dart' as http;

import '../logic/qualified_task.dart';
import '../model/build_status_response.pb.dart';
import '../model/commit_firestore.pb.dart';
import '../model/commit.pb.dart';
import '../model/commit_status.pb.dart';
import '../model/commit_tasks_status.pb.dart';
import '../model/key.pb.dart';
import '../model/task.pb.dart';
import '../model/task_firestore.pb.dart';
import 'cocoon.dart';

/// CocoonService for interacting with flutter/flutter production build data.
Expand All @@ -35,6 +38,27 @@ class AppEngineCocoonService implements CocoonService {
/// This is the base for all API requests to cocoon
static const String _baseApiUrl = 'flutter-dashboard.appspot.com';

/// Json keys from response data.
static const String kCommitAvatar = 'Avatar';
static const String kCommitAuthor = 'Author';
static const String kCommitBranch = 'Branch';
static const String kCommitCreateTimestamp = 'CreateTimestamp';
static const String kCommitDocumentName = 'DocumentName';
static const String kCommitMessage = 'Message';
static const String kCommitRepositoryPath = 'RepositoryPath';
static const String kCommitSha = 'Sha';

static const String kTaskAttempts = 'Attempts';
static const String kTaskBringup = 'Bringup';
static const String kTaskBuildNumber = 'BuildNumber';
static const String kTaskCreateTimestamp = 'CreateTimestamp';
static const String kTaskDocumentName = 'DocumentName';
static const String kTaskEndTimestamp = 'EndTimestamp';
static const String kTaskStartTimestamp = 'StartTimestamp';
static const String kTaskStatus = 'Status';
static const String kTaskTaskNmae = 'TaskName';
static const String kTaskTestFlaky = 'TestFlaky';

final http.Client _client;

@override
Expand Down Expand Up @@ -67,6 +91,38 @@ class AppEngineCocoonService implements CocoonService {
}
}

@override
Future<CocoonResponse<List<CommitTasksStatus>>> fetchCommitStatusesFirestore({
CommitStatus? lastCommitStatus,
String? branch,
required String repo,
}) async {
final Map<String, String?> queryParameters = <String, String?>{
if (lastCommitStatus != null) 'lastCommitKey': lastCommitStatus.commit.key.child.name,
'branch': branch ?? _defaultBranch,
'repo': repo,
};
final Uri getStatusUrl = apiEndpoint('/api/public/get-status-firestore', queryParameters: queryParameters);

/// This endpoint returns JSON [List<CommitTasksStatus>]
final http.Response response = await _client.get(getStatusUrl);

if (response.statusCode != HttpStatus.ok) {
return CocoonResponse<List<CommitTasksStatus>>.error(
'/api/public/get-status-firestore returned ${response.statusCode}',
);
}

try {
final Map<String, dynamic> jsonResponse = jsonDecode(response.body);
return CocoonResponse<List<CommitTasksStatus>>.data(
_commitStatusesFromJsonFirestore(jsonResponse['Statuses']),
);
} catch (error) {
return CocoonResponse<List<CommitTasksStatus>>.error(error.toString());
}
}

@override
Future<CocoonResponse<List<String>>> fetchRepos() async {
final Uri getReposUrl = apiEndpoint('/api/public/repos');
Expand Down Expand Up @@ -207,7 +263,6 @@ class AppEngineCocoonService implements CocoonService {

List<CommitStatus> _commitStatusesFromJson(List<dynamic>? jsonCommitStatuses) {
assert(jsonCommitStatuses != null);
// TODO(chillers): Remove adapter code to just use proto fromJson method. https://github.com/flutter/cocoon/issues/441

final List<CommitStatus> statuses = <CommitStatus>[];

Expand All @@ -224,11 +279,34 @@ class AppEngineCocoonService implements CocoonService {
return statuses;
}

List<CommitTasksStatus> _commitStatusesFromJsonFirestore(List<dynamic>? jsonCommitStatuses) {
assert(jsonCommitStatuses != null);
// TODO(chillers): Remove adapter code to just use proto fromJson method. https://github.com/flutter/cocoon/issues/441

final List<CommitTasksStatus> statuses = <CommitTasksStatus>[];
for (final Map<String, dynamic> jsonCommitStatus in jsonCommitStatuses!) {
final Map<String, dynamic> jsonCommit = jsonCommitStatus['Commit'];

statuses.add(
CommitTasksStatus()
..commit = _commitFromJsonFirestore(jsonCommit)
..branch = _branchFromJsonFirestore(jsonCommit)!
..tasks.addAll(_tasksFromJsonFirestore(jsonCommitStatus['Tasks'])),
);
}

return statuses;
}

String? _branchFromJson(Map<String, dynamic> jsonChecklist) {
final Map<String, dynamic> checklist = jsonChecklist['Checklist'];
return checklist['Branch'] as String?;
}

String? _branchFromJsonFirestore(Map<String, dynamic> jsonCommit) {
return jsonCommit['Branch'] as String?;
}

Commit _commitFromJson(Map<String, dynamic> jsonChecklist) {
final Map<String, dynamic> checklist = jsonChecklist['Checklist'];

Expand All @@ -249,6 +327,21 @@ class AppEngineCocoonService implements CocoonService {
return result;
}

CommitDocument _commitFromJsonFirestore(Map<String, dynamic> jsonCommit) {
final CommitDocument result = CommitDocument()
..documentName = jsonCommit[kCommitDocumentName]
..createTimestamp = Int64(jsonCommit[kCommitCreateTimestamp] as int)
..sha = jsonCommit[kCommitSha] as String
..author = jsonCommit[kCommitAuthor] as String
..avatar = jsonCommit[kCommitAvatar] as String
..repositoryPath = jsonCommit[kCommitRepositoryPath] as String
..branch = jsonCommit[kCommitBranch] as String;
if (jsonCommit[kCommitMessage] != null) {
result.message = jsonCommit[kCommitMessage] as String;
}
return result;
}

List<Task> _tasksFromStagesJson(List<dynamic> json) {
final List<Task> tasks = <Task>[];

Expand All @@ -270,6 +363,17 @@ class AppEngineCocoonService implements CocoonService {
return tasks;
}

List<TaskDocument> _tasksFromJsonFirestore(List<dynamic> json) {
final List<TaskDocument> tasks = <TaskDocument>[];

for (final Map<String, dynamic> jsonTask in json) {
//as Iterable<Map<String, Object>>
tasks.add(_taskFromJsonFirestore(jsonTask));
}

return tasks;
}

Task _taskFromJson(Map<String, dynamic> json) {
final Map<String, dynamic> taskData = json['Task'];
final List<dynamic>? objectRequiredCapabilities = taskData['RequiredCapabilities'] as List<dynamic>?;
Expand All @@ -296,4 +400,21 @@ class AppEngineCocoonService implements CocoonService {
..luciBucket = taskData['LuciBucket'] as String? ?? '';
return task;
}

TaskDocument _taskFromJsonFirestore(Map<String, dynamic> taskData) {
final TaskDocument task = TaskDocument()
..createTimestamp = Int64(taskData[kTaskCreateTimestamp] as int)
..startTimestamp = Int64(taskData[kTaskStartTimestamp] as int)
..endTimestamp = Int64(taskData[kTaskEndTimestamp] as int)
..documentName = taskData[kTaskDocumentName] as String
..taskName = taskData[kTaskTaskNmae] as String
..attempts = taskData[kTaskAttempts] as int
..bringup = taskData[kTaskBringup] as bool
..status = taskData[kTaskStatus] as String
..testFlaky = taskData[kTaskTestFlaky] as bool? ?? false;
if (taskData[kTaskBuildNumber] != null) {
task.buildNumber = taskData[kTaskBuildNumber] as int;
}
return task;
}
}
11 changes: 11 additions & 0 deletions dashboard/lib/service/cocoon.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import 'package:flutter_dashboard/model/branch.pb.dart';

import '../model/build_status_response.pb.dart';
import '../model/commit_status.pb.dart';
import '../model/commit_tasks_status.pb.dart';
import '../model/task.pb.dart';
import 'appengine_cocoon.dart';
import 'dev_cocoon.dart';
Expand Down Expand Up @@ -38,6 +39,16 @@ abstract class CocoonService {
required String repo,
});

/// Gets build information on the most recent commits.
///
/// If [lastCommitStatus] is given, it will return the next page of
/// [List<CommitTasksStatus>] after [lastCommitStatus], not including it.
Future<CocoonResponse<List<CommitTasksStatus>>> fetchCommitStatusesFirestore({
CommitStatus? lastCommitStatus,
String? branch,
required String repo,
});

/// Gets the current build status of flutter/flutter.
Future<CocoonResponse<BuildStatusResponse>> fetchTreeBuildStatus({
String? branch,
Expand Down
11 changes: 11 additions & 0 deletions dashboard/lib/service/dev_cocoon.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import '../logic/qualified_task.dart';
import '../model/build_status_response.pb.dart';
import '../model/commit.pb.dart';
import '../model/commit_status.pb.dart';
import '../model/commit_tasks_status.pb.dart';
import '../model/key.pb.dart';
import '../model/task.pb.dart';
import 'cocoon.dart';
Expand Down Expand Up @@ -68,6 +69,16 @@ class DevelopmentCocoonService implements CocoonService {
_paused = pause;
}

@override
Future<CocoonResponse<List<CommitTasksStatus>>> fetchCommitStatusesFirestore({
CommitStatus? lastCommitStatus,
String? branch,
required String repo,
}) async {
// TODO(keyonghan): to be impelemented when logics are switched to Firestore.
return const CocoonResponse<List<CommitTasksStatus>>.error('');
}

@override
Future<CocoonResponse<List<CommitStatus>>> fetchCommitStatuses({
CommitStatus? lastCommitStatus,
Expand Down
71 changes: 71 additions & 0 deletions dashboard/test/service/appengine_cocoon_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@ import 'package:flutter_dashboard/logic/qualified_task.dart';
import 'package:flutter_dashboard/model/branch.pb.dart';
import 'package:flutter_dashboard/model/build_status_response.pb.dart';
import 'package:flutter_dashboard/model/commit.pb.dart';
import 'package:flutter_dashboard/model/commit_firestore.pb.dart';
import 'package:flutter_dashboard/model/commit_status.pb.dart';
import 'package:flutter_dashboard/model/commit_tasks_status.pb.dart';
import 'package:flutter_dashboard/model/key.pb.dart';
import 'package:flutter_dashboard/model/task.pb.dart';
import 'package:flutter_dashboard/model/task_firestore.pb.dart';
import 'package:flutter_dashboard/service/appengine_cocoon.dart';
import 'package:flutter_dashboard/service/cocoon.dart';
import 'package:flutter_test/flutter_test.dart';
Expand Down Expand Up @@ -92,6 +95,74 @@ void main() {
});
});

group('AppEngine CocoonService fetchCommitStatusFirestore', () {
late AppEngineCocoonService service;

setUp(() async {
service = AppEngineCocoonService(
client: MockClient((Request request) async {
return Response(luciJsonGetStatsResponseFirestore, 200);
}),
);
});

test('should return CocoonResponse<List<CommitStatus>>', () {
expect(
service.fetchCommitStatusesFirestore(repo: 'engine'),
const TypeMatcher<Future<CocoonResponse<List<CommitTasksStatus>>>>(),
);
});

test('should return expected List<CommitTasksStatus>', () async {
final CocoonResponse<List<CommitTasksStatus>> statuses =
await service.fetchCommitStatusesFirestore(repo: 'engine');

final CommitTasksStatus expectedStatus = CommitTasksStatus()
..branch = 'master'
..commit = (CommitDocument()
..documentName = 'commit/document/name'
..createTimestamp = Int64(123456789)
..sha = 'ShaShankHash'
..author = 'ShaSha'
..avatar = 'https://flutter.dev'
..repositoryPath = 'flutter/cocoon'
..branch = 'master'
..message = 'message')
..tasks.add(
TaskDocument()
..documentName = 'task/document/name'
..createTimestamp = Int64(1569353940885)
..startTimestamp = Int64(1569354594672)
..endTimestamp = Int64(1569354700642)
..taskName = 'linux'
..attempts = 1
..bringup = false
..status = 'Succeeded'
..testFlaky = false
..buildNumber = 123,
);

expect(statuses.data!.length, 1);
expect(statuses.data!.first, expectedStatus);
});

test('should have error if given non-200 response', () async {
service = AppEngineCocoonService(client: MockClient((Request request) async => Response('', 404)));

final CocoonResponse<List<CommitTasksStatus>> response =
await service.fetchCommitStatusesFirestore(repo: 'engine');
expect(response.error, isNotNull);
});

test('should have error if given bad response', () async {
service = AppEngineCocoonService(client: MockClient((Request request) async => Response('bad', 200)));

final CocoonResponse<List<CommitTasksStatus>> response =
await service.fetchCommitStatusesFirestore(repo: 'engine');
expect(response.error, isNotNull);
});
});

group('AppEngine CocoonService fetchTreeBuildStatus', () {
late AppEngineCocoonService service;

Expand Down
33 changes: 33 additions & 0 deletions dashboard/test/utils/appengine_cocoon_test_data.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,39 @@ const String luciJsonGetStatsResponse = '''
}
''';

const String luciJsonGetStatsResponseFirestore = '''
{
"Statuses": [
{
"Commit": {
"DocumentName": "commit/document/name",
"Branch": "master",
"RepositoryPath": "flutter/cocoon",
"CreateTimestamp": 123456789,
"Sha": "ShaShankHash",
"Author": "ShaSha",
"Avatar": "https://flutter.dev",
"Message": "message"
},
"Tasks": [
{
"DocumentName": "task/document/name",
"Status": "Succeeded",
"Attempts": 1,
"CreateTimestamp": 1569353940885,
"EndTimestamp": 1569354700642,
"Bringup": false,
"TaskName": "linux",
"StartTimestamp": 1569354594672,
"BuildNumber": 123,
"TestFlaky": false
}
]
}
]
}
''';

const String jsonGetBranchesResponse = '''[
{
"branch":"flutter-3.13-candidate.0",
Expand Down

0 comments on commit bbfd8e8

Please sign in to comment.