Skip to content

Commit

Permalink
Add failure only reporter (#2190)
Browse files Browse the repository at this point in the history
Closes #829

Copies the expanded reporter and test and removes output unrelated to
failed tests.
  • Loading branch information
natebosch authored Mar 4, 2024
1 parent 525f77b commit 7724aab
Show file tree
Hide file tree
Showing 5 changed files with 563 additions and 1 deletion.
2 changes: 1 addition & 1 deletion pkgs/test/pubspec.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ dependencies:

# Use an exact version until the test_api and test_core package are stable.
test_api: 0.7.0
test_core: 0.6.0
test_core: 0.6.1

typed_data: ^1.3.0
web_socket_channel: ^2.0.0
Expand Down
257 changes: 257 additions & 0 deletions pkgs/test/test/runner/failures_only_reporter_test.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,257 @@
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

@TestOn('vm')
library;

import 'dart:async';

import 'package:test/test.dart';
import 'package:test_descriptor/test_descriptor.dart' as d;

import '../io.dart';

void main() {
setUpAll(precompileTestExecutable);

test('reports when no tests are run', () async {
await d.file('test.dart', 'void main() {}').create();

var test = await runTest(['test.dart'], reporter: 'failures-only');
expect(test.stdout, emitsThrough(contains('No tests ran.')));
await test.shouldExit(79);
});

test('runs several successful tests and reports only at the end', () {
return _expectReport('''
test('success 1', () {});
test('success 2', () {});
test('success 3', () {});''', '''
+3: All tests passed!''');
});

test('runs several failing tests and reports when each fails', () {
return _expectReport('''
test('failure 1', () => throw TestFailure('oh no'));
test('failure 2', () => throw TestFailure('oh no'));
test('failure 3', () => throw TestFailure('oh no'));''', '''
+0 -1: failure 1 [E]
oh no
test.dart 6:33 main.<fn>
+0 -2: failure 2 [E]
oh no
test.dart 7:33 main.<fn>
+0 -3: failure 3 [E]
oh no
test.dart 8:33 main.<fn>
+0 -3: Some tests failed.''');
});

test('includes the full stack trace with --verbose-trace', () async {
await d.file('test.dart', '''
import 'dart:async';
import 'package:test/test.dart';
void main() {
test("failure", () => throw "oh no");
}
''').create();

var test = await runTest(['--verbose-trace', 'test.dart'],
reporter: 'failures-only');
expect(test.stdout, emitsThrough(contains('dart:async')));
await test.shouldExit(1);
});

test('reports only failing tests amid successful tests', () {
return _expectReport('''
test('failure 1', () => throw TestFailure('oh no'));
test('success 1', () {});
test('failure 2', () => throw TestFailure('oh no'));
test('success 2', () {});''', '''
+0 -1: failure 1 [E]
oh no
test.dart 6:33 main.<fn>
+1 -2: failure 2 [E]
oh no
test.dart 8:33 main.<fn>
+2 -2: Some tests failed.''');
});

group('print:', () {
test('handles multiple prints', () {
return _expectReport('''
test('test', () {
print("one");
print("two");
print("three");
print("four");
});''', '''
+0: test
one
two
three
four
+1: All tests passed!''');
});

test('handles a print after the test completes', () {
return _expectReport('''
// This completer ensures that the test isolate isn't killed until all
// prints have happened.
var testDone = Completer();
var waitStarted = Completer();
test('test', () async {
waitStarted.future.then((_) {
Future(() => print("one"));
Future(() => print("two"));
Future(() => print("three"));
Future(() => print("four"));
Future(testDone.complete);
});
});
test('wait', () {
waitStarted.complete();
return testDone.future;
});''', '''
+1: test
one
two
three
four
+2: All tests passed!''');
});

test('interleaves prints and errors', () {
return _expectReport('''
// This completer ensures that the test isolate isn't killed until all
// prints have happened.
var completer = Completer();
test('test', () {
scheduleMicrotask(() {
print("three");
print("four");
throw "second error";
});
scheduleMicrotask(() {
print("five");
print("six");
completer.complete();
});
print("one");
print("two");
throw "first error";
});
test('wait', () => completer.future);''', '''
+0: test
one
two
+0 -1: test [E]
first error
test.dart 24:11 main.<fn>
three
four
second error
test.dart 13:13 main.<fn>.<fn>
===== asynchronous gap ===========================
dart:async scheduleMicrotask
test.dart 10:11 main.<fn>
five
six
+1 -1: Some tests failed.''');
});
});

group('skip:', () {
test('does not emit for skips', () {
return _expectReport('''
test('skip 1', () {}, skip: true);
test('skip 2', () {}, skip: true);
test('skip 3', () {}, skip: true);''', '''
+0 ~3: All tests skipped.''');
});

test('runs skipped tests along with successful and failing tests', () {
return _expectReport('''
test('failure 1', () => throw TestFailure('oh no'));
test('skip 1', () {}, skip: true);
test('success 1', () {});
test('failure 2', () => throw TestFailure('oh no'));
test('skip 2', () {}, skip: true);
test('success 2', () {});''', '''
+0 -1: failure 1 [E]
oh no
test.dart 6:35 main.<fn>
+1 ~1 -2: failure 2 [E]
oh no
test.dart 9:35 main.<fn>
+2 ~2 -2: Some tests failed.''');
});
});

test('Directs users to enable stack trace chaining if disabled', () async {
await _expectReport(
'''test('failure 1', () => throw TestFailure('oh no'));''', '''
+0 -1: failure 1 [E]
oh no
test.dart 6:25 main.<fn>
+0 -1: Some tests failed.
Consider enabling the flag chain-stack-traces to receive more detailed exceptions.
For example, 'dart test --chain-stack-traces'.''',
chainStackTraces: false);
});
}

Future<void> _expectReport(String tests, String expected,
{List<String> args = const [], bool chainStackTraces = true}) async {
await d.file('test.dart', '''
import 'dart:async';
import 'package:test/test.dart';
void main() {
$tests
}
''').create();

var test = await runTest([
'test.dart',
if (chainStackTraces) '--chain-stack-traces',
...args,
], reporter: 'failures-only');
await test.shouldExit();

var stdoutLines = await test.stdoutStream().toList();

// Remove excess trailing whitespace.
var actual = stdoutLines.map((line) {
if (line.startsWith(' ') || line.isEmpty) return line.trimRight();
return line.trim();
}).join('\n');

// Un-indent the expected string.
var indentation = expected.indexOf(RegExp('[^ ]'));
expected = expected.split('\n').map((line) {
if (line.isEmpty) return line;
return line.substring(indentation);
}).join('\n');

expect(actual, equals(expected));
}
1 change: 1 addition & 0 deletions pkgs/test/test/runner/runner_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ Output:
[compact] A single line, updated continuously.
[expanded] (default) A separate line for each update.
[failures-only] A separate line for failing tests with no output for passing tests
[github] A custom reporter for GitHub Actions (the default reporter when running on GitHub Actions).
[json] A machine-readable format (see https://dart.dev/go/test-docs/json_reporter.md).
[silent] A reporter with no output. May be useful when only the exit code is meaningful.
Expand Down
9 changes: 9 additions & 0 deletions pkgs/test_core/lib/src/runner/configuration/reporters.dart
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import '../engine.dart';
import '../reporter.dart';
import '../reporter/compact.dart';
import '../reporter/expanded.dart';
import '../reporter/failures_only.dart';
import '../reporter/github.dart';
import '../reporter/json.dart';

Expand Down Expand Up @@ -46,6 +47,14 @@ final _allReporters = <String, ReporterDetails>{
Directory(config.testSelections.keys.single).existsSync(),
printPlatform: config.suiteDefaults.runtimes.length > 1 ||
config.suiteDefaults.compilerSelections != null)),
'failures-only': ReporterDetails(
'A separate line for failing tests with no output for passing tests',
(config, engine, sink) => FailuresOnlyReporter.watch(engine, sink,
color: config.color,
printPath: config.testSelections.length > 1 ||
Directory(config.testSelections.keys.single).existsSync(),
printPlatform: config.suiteDefaults.runtimes.length > 1 ||
config.suiteDefaults.compilerSelections != null)),
'github': ReporterDetails(
'A custom reporter for GitHub Actions '
'(the default reporter when running on GitHub Actions).',
Expand Down
Loading

0 comments on commit 7724aab

Please sign in to comment.