From bbfd8e89c63428f5c9403326bf35521db08b5a51 Mon Sep 17 00:00:00 2001 From: keyonghan <54558023+keyonghan@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:09:04 -0700 Subject: [PATCH] Add logics to fetch commit status from Firestore (#3634) This is part of https://github.com/flutter/flutter/issues/142951 This PR adds logics to fetch commit tasks status from Firestore based on newly added Firestore data model protos. After this lands, next step is to switch the build logic to reference this PR's changes from Datastore. This PR is a no-op change by this point. --- dashboard/lib/service/appengine_cocoon.dart | 123 +++++++++++++++++- dashboard/lib/service/cocoon.dart | 11 ++ dashboard/lib/service/dev_cocoon.dart | 11 ++ .../test/service/appengine_cocoon_test.dart | 71 ++++++++++ .../utils/appengine_cocoon_test_data.dart | 33 +++++ 5 files changed, 248 insertions(+), 1 deletion(-) diff --git a/dashboard/lib/service/appengine_cocoon.dart b/dashboard/lib/service/appengine_cocoon.dart index 5c2dd51b5..2119846df 100644 --- a/dashboard/lib/service/appengine_cocoon.dart +++ b/dashboard/lib/service/appengine_cocoon.dart @@ -12,10 +12,13 @@ import 'package:http/http.dart' as http; import '../logic/qualified_task.dart'; import '../model/build_status_response.pb.dart'; +import '../model/commit_firestore.pb.dart'; import '../model/commit.pb.dart'; import '../model/commit_status.pb.dart'; +import '../model/commit_tasks_status.pb.dart'; import '../model/key.pb.dart'; import '../model/task.pb.dart'; +import '../model/task_firestore.pb.dart'; import 'cocoon.dart'; /// CocoonService for interacting with flutter/flutter production build data. @@ -35,6 +38,27 @@ class AppEngineCocoonService implements CocoonService { /// This is the base for all API requests to cocoon static const String _baseApiUrl = 'flutter-dashboard.appspot.com'; + /// Json keys from response data. + static const String kCommitAvatar = 'Avatar'; + static const String kCommitAuthor = 'Author'; + static const String kCommitBranch = 'Branch'; + static const String kCommitCreateTimestamp = 'CreateTimestamp'; + static const String kCommitDocumentName = 'DocumentName'; + static const String kCommitMessage = 'Message'; + static const String kCommitRepositoryPath = 'RepositoryPath'; + static const String kCommitSha = 'Sha'; + + static const String kTaskAttempts = 'Attempts'; + static const String kTaskBringup = 'Bringup'; + static const String kTaskBuildNumber = 'BuildNumber'; + static const String kTaskCreateTimestamp = 'CreateTimestamp'; + static const String kTaskDocumentName = 'DocumentName'; + static const String kTaskEndTimestamp = 'EndTimestamp'; + static const String kTaskStartTimestamp = 'StartTimestamp'; + static const String kTaskStatus = 'Status'; + static const String kTaskTaskNmae = 'TaskName'; + static const String kTaskTestFlaky = 'TestFlaky'; + final http.Client _client; @override @@ -67,6 +91,38 @@ class AppEngineCocoonService implements CocoonService { } } + @override + Future>> fetchCommitStatusesFirestore({ + CommitStatus? lastCommitStatus, + String? branch, + required String repo, + }) async { + final Map queryParameters = { + if (lastCommitStatus != null) 'lastCommitKey': lastCommitStatus.commit.key.child.name, + 'branch': branch ?? _defaultBranch, + 'repo': repo, + }; + final Uri getStatusUrl = apiEndpoint('/api/public/get-status-firestore', queryParameters: queryParameters); + + /// This endpoint returns JSON [List] + final http.Response response = await _client.get(getStatusUrl); + + if (response.statusCode != HttpStatus.ok) { + return CocoonResponse>.error( + '/api/public/get-status-firestore returned ${response.statusCode}', + ); + } + + try { + final Map jsonResponse = jsonDecode(response.body); + return CocoonResponse>.data( + _commitStatusesFromJsonFirestore(jsonResponse['Statuses']), + ); + } catch (error) { + return CocoonResponse>.error(error.toString()); + } + } + @override Future>> fetchRepos() async { final Uri getReposUrl = apiEndpoint('/api/public/repos'); @@ -207,7 +263,6 @@ class AppEngineCocoonService implements CocoonService { List _commitStatusesFromJson(List? jsonCommitStatuses) { assert(jsonCommitStatuses != null); - // TODO(chillers): Remove adapter code to just use proto fromJson method. https://github.com/flutter/cocoon/issues/441 final List statuses = []; @@ -224,11 +279,34 @@ class AppEngineCocoonService implements CocoonService { return statuses; } + List _commitStatusesFromJsonFirestore(List? jsonCommitStatuses) { + assert(jsonCommitStatuses != null); + // TODO(chillers): Remove adapter code to just use proto fromJson method. https://github.com/flutter/cocoon/issues/441 + + final List statuses = []; + for (final Map jsonCommitStatus in jsonCommitStatuses!) { + final Map jsonCommit = jsonCommitStatus['Commit']; + + statuses.add( + CommitTasksStatus() + ..commit = _commitFromJsonFirestore(jsonCommit) + ..branch = _branchFromJsonFirestore(jsonCommit)! + ..tasks.addAll(_tasksFromJsonFirestore(jsonCommitStatus['Tasks'])), + ); + } + + return statuses; + } + String? _branchFromJson(Map jsonChecklist) { final Map checklist = jsonChecklist['Checklist']; return checklist['Branch'] as String?; } + String? _branchFromJsonFirestore(Map jsonCommit) { + return jsonCommit['Branch'] as String?; + } + Commit _commitFromJson(Map jsonChecklist) { final Map checklist = jsonChecklist['Checklist']; @@ -249,6 +327,21 @@ class AppEngineCocoonService implements CocoonService { return result; } + CommitDocument _commitFromJsonFirestore(Map jsonCommit) { + final CommitDocument result = CommitDocument() + ..documentName = jsonCommit[kCommitDocumentName] + ..createTimestamp = Int64(jsonCommit[kCommitCreateTimestamp] as int) + ..sha = jsonCommit[kCommitSha] as String + ..author = jsonCommit[kCommitAuthor] as String + ..avatar = jsonCommit[kCommitAvatar] as String + ..repositoryPath = jsonCommit[kCommitRepositoryPath] as String + ..branch = jsonCommit[kCommitBranch] as String; + if (jsonCommit[kCommitMessage] != null) { + result.message = jsonCommit[kCommitMessage] as String; + } + return result; + } + List _tasksFromStagesJson(List json) { final List tasks = []; @@ -270,6 +363,17 @@ class AppEngineCocoonService implements CocoonService { return tasks; } + List _tasksFromJsonFirestore(List json) { + final List tasks = []; + + for (final Map jsonTask in json) { + //as Iterable> + tasks.add(_taskFromJsonFirestore(jsonTask)); + } + + return tasks; + } + Task _taskFromJson(Map json) { final Map taskData = json['Task']; final List? objectRequiredCapabilities = taskData['RequiredCapabilities'] as List?; @@ -296,4 +400,21 @@ class AppEngineCocoonService implements CocoonService { ..luciBucket = taskData['LuciBucket'] as String? ?? ''; return task; } + + TaskDocument _taskFromJsonFirestore(Map taskData) { + final TaskDocument task = TaskDocument() + ..createTimestamp = Int64(taskData[kTaskCreateTimestamp] as int) + ..startTimestamp = Int64(taskData[kTaskStartTimestamp] as int) + ..endTimestamp = Int64(taskData[kTaskEndTimestamp] as int) + ..documentName = taskData[kTaskDocumentName] as String + ..taskName = taskData[kTaskTaskNmae] as String + ..attempts = taskData[kTaskAttempts] as int + ..bringup = taskData[kTaskBringup] as bool + ..status = taskData[kTaskStatus] as String + ..testFlaky = taskData[kTaskTestFlaky] as bool? ?? false; + if (taskData[kTaskBuildNumber] != null) { + task.buildNumber = taskData[kTaskBuildNumber] as int; + } + return task; + } } diff --git a/dashboard/lib/service/cocoon.dart b/dashboard/lib/service/cocoon.dart index 0a40b7db6..e415e21fa 100644 --- a/dashboard/lib/service/cocoon.dart +++ b/dashboard/lib/service/cocoon.dart @@ -7,6 +7,7 @@ import 'package:flutter_dashboard/model/branch.pb.dart'; import '../model/build_status_response.pb.dart'; import '../model/commit_status.pb.dart'; +import '../model/commit_tasks_status.pb.dart'; import '../model/task.pb.dart'; import 'appengine_cocoon.dart'; import 'dev_cocoon.dart'; @@ -38,6 +39,16 @@ abstract class CocoonService { required String repo, }); + /// Gets build information on the most recent commits. + /// + /// If [lastCommitStatus] is given, it will return the next page of + /// [List] after [lastCommitStatus], not including it. + Future>> fetchCommitStatusesFirestore({ + CommitStatus? lastCommitStatus, + String? branch, + required String repo, + }); + /// Gets the current build status of flutter/flutter. Future> fetchTreeBuildStatus({ String? branch, diff --git a/dashboard/lib/service/dev_cocoon.dart b/dashboard/lib/service/dev_cocoon.dart index 3a9ad5a46..aa6fde098 100644 --- a/dashboard/lib/service/dev_cocoon.dart +++ b/dashboard/lib/service/dev_cocoon.dart @@ -12,6 +12,7 @@ import '../logic/qualified_task.dart'; import '../model/build_status_response.pb.dart'; import '../model/commit.pb.dart'; import '../model/commit_status.pb.dart'; +import '../model/commit_tasks_status.pb.dart'; import '../model/key.pb.dart'; import '../model/task.pb.dart'; import 'cocoon.dart'; @@ -68,6 +69,16 @@ class DevelopmentCocoonService implements CocoonService { _paused = pause; } + @override + Future>> fetchCommitStatusesFirestore({ + CommitStatus? lastCommitStatus, + String? branch, + required String repo, + }) async { + // TODO(keyonghan): to be impelemented when logics are switched to Firestore. + return const CocoonResponse>.error(''); + } + @override Future>> fetchCommitStatuses({ CommitStatus? lastCommitStatus, diff --git a/dashboard/test/service/appengine_cocoon_test.dart b/dashboard/test/service/appengine_cocoon_test.dart index e7a6ca745..39f547de6 100644 --- a/dashboard/test/service/appengine_cocoon_test.dart +++ b/dashboard/test/service/appengine_cocoon_test.dart @@ -8,9 +8,12 @@ import 'package:flutter_dashboard/logic/qualified_task.dart'; import 'package:flutter_dashboard/model/branch.pb.dart'; import 'package:flutter_dashboard/model/build_status_response.pb.dart'; import 'package:flutter_dashboard/model/commit.pb.dart'; +import 'package:flutter_dashboard/model/commit_firestore.pb.dart'; import 'package:flutter_dashboard/model/commit_status.pb.dart'; +import 'package:flutter_dashboard/model/commit_tasks_status.pb.dart'; import 'package:flutter_dashboard/model/key.pb.dart'; import 'package:flutter_dashboard/model/task.pb.dart'; +import 'package:flutter_dashboard/model/task_firestore.pb.dart'; import 'package:flutter_dashboard/service/appengine_cocoon.dart'; import 'package:flutter_dashboard/service/cocoon.dart'; import 'package:flutter_test/flutter_test.dart'; @@ -92,6 +95,74 @@ void main() { }); }); + group('AppEngine CocoonService fetchCommitStatusFirestore', () { + late AppEngineCocoonService service; + + setUp(() async { + service = AppEngineCocoonService( + client: MockClient((Request request) async { + return Response(luciJsonGetStatsResponseFirestore, 200); + }), + ); + }); + + test('should return CocoonResponse>', () { + expect( + service.fetchCommitStatusesFirestore(repo: 'engine'), + const TypeMatcher>>>(), + ); + }); + + test('should return expected List', () async { + final CocoonResponse> statuses = + await service.fetchCommitStatusesFirestore(repo: 'engine'); + + final CommitTasksStatus expectedStatus = CommitTasksStatus() + ..branch = 'master' + ..commit = (CommitDocument() + ..documentName = 'commit/document/name' + ..createTimestamp = Int64(123456789) + ..sha = 'ShaShankHash' + ..author = 'ShaSha' + ..avatar = 'https://flutter.dev' + ..repositoryPath = 'flutter/cocoon' + ..branch = 'master' + ..message = 'message') + ..tasks.add( + TaskDocument() + ..documentName = 'task/document/name' + ..createTimestamp = Int64(1569353940885) + ..startTimestamp = Int64(1569354594672) + ..endTimestamp = Int64(1569354700642) + ..taskName = 'linux' + ..attempts = 1 + ..bringup = false + ..status = 'Succeeded' + ..testFlaky = false + ..buildNumber = 123, + ); + + expect(statuses.data!.length, 1); + expect(statuses.data!.first, expectedStatus); + }); + + test('should have error if given non-200 response', () async { + service = AppEngineCocoonService(client: MockClient((Request request) async => Response('', 404))); + + final CocoonResponse> response = + await service.fetchCommitStatusesFirestore(repo: 'engine'); + expect(response.error, isNotNull); + }); + + test('should have error if given bad response', () async { + service = AppEngineCocoonService(client: MockClient((Request request) async => Response('bad', 200))); + + final CocoonResponse> response = + await service.fetchCommitStatusesFirestore(repo: 'engine'); + expect(response.error, isNotNull); + }); + }); + group('AppEngine CocoonService fetchTreeBuildStatus', () { late AppEngineCocoonService service; diff --git a/dashboard/test/utils/appengine_cocoon_test_data.dart b/dashboard/test/utils/appengine_cocoon_test_data.dart index 22cd8adf9..663e79152 100644 --- a/dashboard/test/utils/appengine_cocoon_test_data.dart +++ b/dashboard/test/utils/appengine_cocoon_test_data.dart @@ -57,6 +57,39 @@ const String luciJsonGetStatsResponse = ''' } '''; +const String luciJsonGetStatsResponseFirestore = ''' + { + "Statuses": [ + { + "Commit": { + "DocumentName": "commit/document/name", + "Branch": "master", + "RepositoryPath": "flutter/cocoon", + "CreateTimestamp": 123456789, + "Sha": "ShaShankHash", + "Author": "ShaSha", + "Avatar": "https://flutter.dev", + "Message": "message" + }, + "Tasks": [ + { + "DocumentName": "task/document/name", + "Status": "Succeeded", + "Attempts": 1, + "CreateTimestamp": 1569353940885, + "EndTimestamp": 1569354700642, + "Bringup": false, + "TaskName": "linux", + "StartTimestamp": 1569354594672, + "BuildNumber": 123, + "TestFlaky": false + } + ] + } + ] + } +'''; + const String jsonGetBranchesResponse = '''[ { "branch":"flutter-3.13-candidate.0",