Run a subset of tests on the Flutter customer test registry. (#8552)

flutter · Nov 26, 2024 · d58891c · d58891c
1 parent c9ea8af
commit d58891c
Show file tree

Hide file tree

Showing 5 changed files with 160 additions and 115 deletions.
diff --git a/packages/devtools_app/test/debugger/debugger_evaluation_test.dart b/packages/devtools_app/test/debugger/debugger_evaluation_test.dart
@@ -86,6 +86,7 @@ void main() {
         );
       },
       timeout: const Timeout.factor(8),
+      tags: includeForCustomerTestsTag,
     );
 
     test(
@@ -110,6 +111,7 @@ void main() {
         );
       },
       timeout: const Timeout.factor(8),
+      tags: includeForCustomerTestsTag,
     );
 
     test(
@@ -144,6 +146,7 @@ void main() {
         );
       },
       timeout: const Timeout.factor(8),
+      tags: includeForCustomerTestsTag,
     );
     test(
       'returns privates only from library',
@@ -169,129 +172,145 @@ void main() {
         );
       },
       timeout: const Timeout.factor(8),
+      tags: includeForCustomerTestsTag,
     );
-    test('returns exported members from import', () async {
-      await runMethodAndWaitForPause(
-        'AnotherClass().pauseWithScopedVariablesMethod()',
-      );
-      expect(
-        await autoCompleteResultsFor(
-          EditingParts(
-            activeWord: 'exportedField',
-            leftSide: '',
-            rightSide: '',
+    test(
+      'returns exported members from import',
+      () async {
+        await runMethodAndWaitForPause(
+          'AnotherClass().pauseWithScopedVariablesMethod()',
+        );
+        expect(
+          await autoCompleteResultsFor(
+            EditingParts(
+              activeWord: 'exportedField',
+              leftSide: '',
+              rightSide: '',
+            ),
+            evalService,
           ),
-          evalService,
-        ),
-        equals(['exportedField']),
-      );
+          equals(['exportedField']),
+        );
 
-      expect(
-        await autoCompleteResultsFor(
-          EditingParts(
-            activeWord: 'ExportedClass',
-            leftSide: '',
-            rightSide: '',
+        expect(
+          await autoCompleteResultsFor(
+            EditingParts(
+              activeWord: 'ExportedClass',
+              leftSide: '',
+              rightSide: '',
+            ),
+            evalService,
           ),
-          evalService,
-        ),
-        equals(['ExportedClass']),
-      );
+          equals(['ExportedClass']),
+        );
 
-      // Privates are not exported
-      expect(
-        await autoCompleteResultsFor(
-          EditingParts(
-            activeWord: '_privateExportedField',
-            leftSide: '',
-            rightSide: '',
+        // Privates are not exported
+        expect(
+          await autoCompleteResultsFor(
+            EditingParts(
+              activeWord: '_privateExportedField',
+              leftSide: '',
+              rightSide: '',
+            ),
+            evalService,
           ),
-          evalService,
-        ),
-        equals([]),
-      );
+          equals([]),
+        );
 
-      expect(
-        await autoCompleteResultsFor(
-          EditingParts(
-            activeWord: '_PrivateExportedClass',
-            leftSide: '',
-            rightSide: '',
+        expect(
+          await autoCompleteResultsFor(
+            EditingParts(
+              activeWord: '_PrivateExportedClass',
+              leftSide: '',
+              rightSide: '',
+            ),
+            evalService,
           ),
-          evalService,
-        ),
-        equals([]),
-      );
-    }, timeout: const Timeout.factor(8));
+          equals([]),
+        );
+      },
+      timeout: const Timeout.factor(8),
+      tags: includeForCustomerTestsTag,
+    );
 
-    test('returns prefixes of libraries imported', () async {
-      await runMethodAndWaitForPause(
-        'AnotherClass().pauseWithScopedVariablesMethod()',
-      );
-      expect(
-        await autoCompleteResultsFor(
-          EditingParts(activeWord: 'developer', leftSide: '', rightSide: ''),
-          evalService,
-        ),
-        equals(['developer']),
-      );
+    test(
+      'returns prefixes of libraries imported',
+      () async {
+        await runMethodAndWaitForPause(
+          'AnotherClass().pauseWithScopedVariablesMethod()',
+        );
+        expect(
+          await autoCompleteResultsFor(
+            EditingParts(activeWord: 'developer', leftSide: '', rightSide: ''),
+            evalService,
+          ),
+          equals(['developer']),
+        );
 
-      expect(
-        await autoCompleteResultsFor(
-          EditingParts(activeWord: 'math', leftSide: '', rightSide: ''),
-          evalService,
-        ),
-        equals(['math']),
-      );
-    }, timeout: const Timeout.factor(8));
+        expect(
+          await autoCompleteResultsFor(
+            EditingParts(activeWord: 'math', leftSide: '', rightSide: ''),
+            evalService,
+          ),
+          equals(['math']),
+        );
+      },
+      timeout: const Timeout.factor(8),
+      tags: includeForCustomerTestsTag,
+    );
 
-    test('returns no operators for int', () async {
-      await runMethodAndWaitForPause(
-        'AnotherClass().pauseWithScopedVariablesMethod()',
-      );
-      expect(
-        await autoCompleteResultsFor(
-          EditingParts(leftSide: '7.', activeWord: '', rightSide: ''),
-          evalService,
-        ),
-        equals([
-          'hashCode',
-          'bitLength',
-          'toString',
-          'remainder',
-          'abs',
-          'sign',
-          'isEven',
-          'isOdd',
-          'isNaN',
-          'isNegative',
-          'isInfinite',
-          'isFinite',
-          'toUnsigned',
-          'toSigned',
-          'compareTo',
-          'round',
-          'floor',
-          'ceil',
-          'truncate',
-          'roundToDouble',
-          'floorToDouble',
-          'ceilToDouble',
-          'truncateToDouble',
-          'clamp',
-          'toInt',
-          'toDouble',
-          'toStringAsFixed',
-          'toStringAsExponential',
-          'toStringAsPrecision',
-          'toRadixString',
-          'modPow',
-          'modInverse',
-          'gcd',
-          'noSuchMethod',
-          'runtimeType',
-        ]),
-      );
-    }, timeout: const Timeout.factor(8));
+    test(
+      'returns no operators for int',
+      () async {
+        await runMethodAndWaitForPause(
+          'AnotherClass().pauseWithScopedVariablesMethod()',
+        );
+        expect(
+          await autoCompleteResultsFor(
+            EditingParts(leftSide: '7.', activeWord: '', rightSide: ''),
+            evalService,
+          ),
+          equals([
+            'hashCode',
+            'bitLength',
+            'toString',
+            'remainder',
+            'abs',
+            'sign',
+            'isEven',
+            'isOdd',
+            'isNaN',
+            'isNegative',
+            'isInfinite',
+            'isFinite',
+            'toUnsigned',
+            'toSigned',
+            'compareTo',
+            'round',
+            'floor',
+            'ceil',
+            'truncate',
+            'roundToDouble',
+            'floorToDouble',
+            'ceilToDouble',
+            'truncateToDouble',
+            'clamp',
+            'toInt',
+            'toDouble',
+            'toStringAsFixed',
+            'toStringAsExponential',
+            'toStringAsPrecision',
+            'toRadixString',
+            'modPow',
+            'modInverse',
+            'gcd',
+            'noSuchMethod',
+            'runtimeType',
+          ]),
+        );
+      },
+      timeout: const Timeout.factor(8),
+      tags: includeForCustomerTestsTag,
+    );
   });
 }
diff --git a/packages/devtools_app/test/test_infra/scenes/memory/default.dart b/packages/devtools_app/test/test_infra/scenes/memory/default.dart
@@ -60,7 +60,7 @@ abstract class MemoryDefaultSceneHeaps {
           .toList();
 
   static final golden =
-          goldenHeapTests
+      goldenHeapTests
           // ignore: avoid-redundant-async, match signature
           .map((e) => () async => e.loadHeap())
           .toList();

diff --git a/packages/devtools_test/lib/src/helpers/utils.dart b/packages/devtools_test/lib/src/helpers/utils.dart
@@ -12,6 +12,22 @@ import 'package:flutter/services.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:path/path.dart' as path;
 
+/// The tag to add to a test case to ensure it is run on each commit to the
+/// Flutter SDK.
+/// 
+/// Before adding this tag, first check if the test is included in one of the
+/// tested subdirectories defined by tool/ci/flutter_customer_tests/test.sh. If
+/// it is, there is no need to add the tag to the individual test case since the
+/// library containing the test case is already included.
+const includeForCustomerTestsTag = 'include-for-flutter-customer-tests';
+
+/// The tag to add to a test case to ensure it is not run on each commit to the
+/// Flutter SDK.
+/// 
+/// Before adding this tag, first check if the test is included in one of the
+/// tested subdirectories defined by tool/ci/flutter_customer_tests/test.sh. If
+/// it is not, there is no need to add this tag to the individual test case
+/// since the library containing the test case is already excluded.
 const skipForCustomerTestsTag = 'skip-for-flutter-customer-tests';
 
 const shortPumpDuration = Duration(seconds: 1);

diff --git a/tool/flutter_customer_tests/README.md b/tool/flutter_customer_tests/README.md
@@ -7,3 +7,10 @@ skip the test case by adding the tag 'skip-for-flutter-customer-tests', which
 is defined in a const in
 [utils.dart](https://github.com/flutter/devtools/blob/master/packages/devtools_test/lib/src/helpers/utils.dart#L15)
 inside the `devtools_test` package.
+
+If a new DevTools test should be run on the Flutter customer test run that is **not**
+already part of one of the `test/` subdirectories tested in `test.sh`, you can add
+the tag 'include-for-flutter-customer-tests', which
+is defined in a const in
+[utils.dart](https://github.com/flutter/devtools/blob/master/packages/devtools_test/lib/src/helpers/utils.dart#L15)
+inside the `devtools_test` package.
diff --git a/tool/flutter_customer_tests/test.sh b/tool/flutter_customer_tests/test.sh
@@ -13,4 +13,7 @@ flutter test test/
 
 cd ../devtools_app
 flutter pub get
-flutter test --exclude-tags=skip-for-flutter-customer-tests test/
+flutter test --tags=include-for-flutter-customer-tests test/
+flutter test --exclude-tags=skip-for-flutter-customer-tests test/inspector/
+flutter test --exclude-tags=skip-for-flutter-customer-tests test/inspector_v2/
+flutter test --exclude-tags=skip-for-flutter-customer-tests test/shared/