diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index 46a069b00..0855aa7ea 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.29.0-wip + +* Add `--coverage-package` flag, which filters the coverage report to specific + packages using RegExps. + ## 1.28.0 * Add `isSorted` and related matchers for iterables. diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml index 9858e925d..1f8d8db60 100644 --- a/pkgs/test/pubspec.yaml +++ b/pkgs/test/pubspec.yaml @@ -1,5 +1,5 @@ name: test -version: 1.28.0 +version: 1.29.0-wip description: >- A full featured library for writing and running Dart tests across platforms. repository: https://github.com/dart-lang/test/tree/master/pkgs/test @@ -36,7 +36,7 @@ dependencies: # Use an exact version until the test_api and test_core package are stable. test_api: 0.7.8 - test_core: 0.6.14 + test_core: 0.6.15-wip typed_data: ^1.3.0 web_socket_channel: '>=2.0.0 <4.0.0' diff --git a/pkgs/test/test/io.dart b/pkgs/test/test/io.dart index cc850fd87..8ad784e08 100644 --- a/pkgs/test/test/io.dart +++ b/pkgs/test/test/io.dart @@ -100,6 +100,7 @@ Future runTest( bool forwardStdio = false, String? packageConfig, Iterable? vmArgs, + String? workingDirectory, }) async { concurrency ??= 1; var testExecutablePath = _testExecutablePath; @@ -127,6 +128,7 @@ Future runTest( description: 'dart bin/test.dart', forwardStdio: forwardStdio, packageConfig: packageConfig, + workingDirectory: workingDirectory, ); } @@ -140,6 +142,7 @@ Future runDart( String? description, bool forwardStdio = false, String? packageConfig, + String? workingDirectory, }) async { var allArgs = [ '--packages=${packageConfig ?? await Isolate.packageConfig}', @@ -149,7 +152,7 @@ Future runDart( return await TestProcess.start( p.absolute(Platform.resolvedExecutable), allArgs, - workingDirectory: d.sandbox, + workingDirectory: workingDirectory ?? d.sandbox, environment: environment, description: description, forwardStdio: forwardStdio, @@ -160,11 +163,12 @@ Future runDart( Future runPub( Iterable args, { Map? environment, + String? workingDirectory, }) { return TestProcess.start( p.absolute(Platform.resolvedExecutable), ['pub', ...args], - workingDirectory: d.sandbox, + workingDirectory: workingDirectory ?? d.sandbox, environment: environment, description: 'pub ${args.first}', ); diff --git a/pkgs/test/test/runner/coverage_test.dart b/pkgs/test/test/runner/coverage_test.dart index 62dce565f..222e76ced 100644 --- a/pkgs/test/test/runner/coverage_test.dart +++ b/pkgs/test/test/runner/coverage_test.dart @@ -21,7 +21,7 @@ void main() { group('with the --coverage flag,', () { late Directory coverageDirectory; - late d.DirectoryDescriptor packageDirectory; + late String pkgDir; Future validateTest(TestProcess test) async { expect(test.stdout, emitsThrough(contains('+1: All tests passed!'))); @@ -47,9 +47,10 @@ void main() { 'test_coverage', ); - packageDirectory = d.dir(d.sandbox, [ - d.dir('lib', [ - d.file('calculate.dart', ''' + final outerDirectory = d.dir(d.sandbox, [ + d.dir('fake_package', [ + d.dir('lib', [ + d.file('calculate.dart', ''' int calculate(int x) { if (x % 2 == 0) { return x * 2; @@ -58,9 +59,9 @@ void main() { } } '''), - ]), - d.dir('test', [ - d.file('test.dart', ''' + ]), + d.dir('test', [ + d.file('test.dart', ''' import 'package:fake_package/calculate.dart'; import 'package:test/test.dart'; @@ -70,8 +71,8 @@ void main() { }); } '''), - ]), - d.file('pubspec.yaml', ''' + ]), + d.file('pubspec.yaml', ''' name: fake_package version: 1.0.0 environment: @@ -79,8 +80,44 @@ environment: dev_dependencies: test: ^1.26.2 '''), + ]), + d.dir('fake_client', [ + d.dir('lib', [ + d.file('calculate.dart', ''' + import 'package:fake_package/calculate.dart' as fake_package; + + int calculate(int x) { + return fake_package.calculate(x); + } + '''), + ]), + d.dir('test', [ + d.file('test.dart', ''' + import 'package:fake_client/calculate.dart'; + import 'package:test/test.dart'; + + void main() { + test('test 1', () { + expect(calculate(6), 12); + }); + } + '''), + ]), + d.file('pubspec.yaml', ''' +name: fake_client +version: 1.0.0 +environment: + sdk: ^3.5.0 +dependencies: + fake_package: + path: ../fake_package +dev_dependencies: + test: ^1.26.2 + '''), + ]), ]); - await packageDirectory.create(); + await outerDirectory.create(); + pkgDir = p.join(d.sandbox, 'fake_package'); }); tearDown(() async { @@ -88,12 +125,12 @@ dev_dependencies: }); test('gathers coverage for VM tests', () async { - await (await runPub(['get'])).shouldExit(0); - var test = await runTest([ - '--coverage', - coverageDirectory.path, - 'test/test.dart', - ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json')); + await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0); + var test = await runTest( + ['--coverage', coverageDirectory.path, 'test/test.dart'], + packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'), + workingDirectory: pkgDir, + ); final coverage = await validateCoverage(test, 'test/test.dart.vm.json'); final hitmap = coverage['package:fake_package/calculate.dart']!; expect(hitmap.lineHits, {1: 1, 2: 2, 3: 1, 5: 0}); @@ -102,7 +139,7 @@ dev_dependencies: }); test('gathers branch coverage for VM tests', () async { - await (await runPub(['get'])).shouldExit(0); + await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0); var test = await runTest( [ '--coverage', @@ -111,7 +148,8 @@ dev_dependencies: 'test/test.dart', ], vmArgs: ['--branch-coverage'], - packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'), + packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'), + workingDirectory: pkgDir, ); final coverage = await validateCoverage(test, 'test/test.dart.vm.json'); final hitmap = coverage['package:fake_package/calculate.dart']!; @@ -121,16 +159,51 @@ dev_dependencies: }); test('gathers lcov coverage for VM tests', () async { - await (await runPub(['get'])).shouldExit(0); + await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0); final lcovFile = p.join(coverageDirectory.path, 'lcov.info'); - var test = await runTest([ - '--coverage-path', - lcovFile, - 'test/test.dart', - ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json')); + var test = await runTest( + ['--coverage-path', lcovFile, 'test/test.dart'], + packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'), + workingDirectory: pkgDir, + ); await validateTest(test); expect(File(lcovFile).readAsStringSync(), ''' -SF:${p.join(d.sandbox, 'lib', 'calculate.dart')} +SF:${p.join(pkgDir, 'lib', 'calculate.dart')} +DA:1,1 +DA:2,2 +DA:3,1 +DA:5,0 +LF:4 +LH:3 +end_of_record +'''); + }); + + test('gathers coverage for tests in multiple pacakges', () async { + final clientPkgDir = p.join(d.sandbox, 'fake_client'); + await (await runPub([ + 'get', + ], workingDirectory: clientPkgDir)).shouldExit(0); + final lcovFile = p.join(coverageDirectory.path, 'lcov.info'); + var test = await runTest( + [ + '--coverage-path', + lcovFile, + '--coverage-package=fake_.*', + 'test/test.dart', + ], + packageConfig: p.join(clientPkgDir, '.dart_tool/package_config.json'), + workingDirectory: clientPkgDir, + ); + await validateTest(test); + expect(File(lcovFile).readAsStringSync(), ''' +SF:${p.join(clientPkgDir, 'lib', 'calculate.dart')} +DA:3,1 +DA:4,1 +LF:2 +LH:2 +end_of_record +SF:${p.join(pkgDir, 'lib', 'calculate.dart')} DA:1,1 DA:2,2 DA:3,1 @@ -142,14 +215,18 @@ end_of_record }); test('gathers coverage for Chrome tests', () async { - await (await runPub(['get'])).shouldExit(0); - var test = await runTest([ - '--coverage', - coverageDirectory.path, - 'test/test.dart', - '-p', - 'chrome', - ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json')); + await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0); + var test = await runTest( + [ + '--coverage', + coverageDirectory.path, + 'test/test.dart', + '-p', + 'chrome', + ], + packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'), + workingDirectory: pkgDir, + ); await validateCoverage(test, 'test/test.dart.chrome.json'); }); diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index fdb4c5738..a42d47611 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart @@ -96,6 +96,9 @@ $_runtimeCompilers Implies --debug. --branch-coverage Include branch coverage information in the coverage report. Must be paired with --coverage or --coverage-path. + --coverage-package= A regular expression matching packages names to include in + the coverage report (if coverage is enabled). If unset, + matches the current package name. --[no-]chain-stack-traces Use chained stack traces to provide greater exception details especially for asynchronous code. It may be useful to disable to provide improved test performance but at the cost of diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart index cfa26cff2..18cae85e8 100644 --- a/pkgs/test/test/utils.dart +++ b/pkgs/test/test/utils.dart @@ -226,6 +226,7 @@ Configuration configuration({ String? coverage, String? coverageLcov, bool? branchCoverage, + List? coveragePackages, int? concurrency, int? shardIndex, int? totalShards, @@ -278,6 +279,7 @@ Configuration configuration({ coverage: coverage, coverageLcov: coverageLcov, branchCoverage: branchCoverage, + coveragePackages: coveragePackages, concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 6709f9704..facfaf1ae 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md @@ -1,3 +1,8 @@ +## 0.6.15-wip + +* Add `--coverage-package` flag, which filters the coverage report to specific + packages using RegExps. + ## 0.6.14 * Fix type cast when parsing a `null` hit map. diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart index e9c3bebe9..236335aba 100644 --- a/pkgs/test_core/lib/src/runner/configuration.dart +++ b/pkgs/test_core/lib/src/runner/configuration.dart @@ -75,6 +75,13 @@ class Configuration { bool get branchCoverage => _branchCoverage ?? false; final bool? _branchCoverage; + /// A list of regular expressions to match against the package names during + /// coverage collection. + /// + /// Only packages that match at least one of the regexps will be included in + /// the coverage report. + final List? coveragePackages; + /// The path to the file from which to load more configuration information. /// /// This is *not* resolved automatically. @@ -273,6 +280,7 @@ class Configuration { required String? coverage, required String? coverageLcov, required bool? branchCoverage, + required List? coveragePackages, required int? concurrency, required int? shardIndex, required int? totalShards, @@ -328,6 +336,7 @@ class Configuration { coverage: coverage, coverageLcov: coverageLcov, branchCoverage: branchCoverage, + coveragePackages: coveragePackages, concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, @@ -389,6 +398,7 @@ class Configuration { String? coverage, String? coverageLcov, bool? branchCoverage, + List? coveragePackages, int? concurrency, int? shardIndex, int? totalShards, @@ -442,6 +452,7 @@ class Configuration { coverage: coverage, coverageLcov: coverageLcov, branchCoverage: branchCoverage, + coveragePackages: coveragePackages, concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, @@ -512,6 +523,7 @@ class Configuration { coverage: null, coverageLcov: null, branchCoverage: null, + coveragePackages: null, concurrency: null, shardIndex: null, totalShards: null, @@ -578,6 +590,7 @@ class Configuration { coverage: null, coverageLcov: null, branchCoverage: null, + coveragePackages: null, concurrency: null, shardIndex: null, totalShards: null, @@ -647,6 +660,7 @@ class Configuration { coverage: null, coverageLcov: null, branchCoverage: null, + coveragePackages: null, shardIndex: null, totalShards: null, testSelections: null, @@ -710,6 +724,7 @@ class Configuration { coverage: null, coverageLcov: null, branchCoverage: null, + coveragePackages: null, concurrency: null, shardIndex: null, totalShards: null, @@ -779,6 +794,7 @@ class Configuration { required this.coverage, required this.coverageLcov, required bool? branchCoverage, + required this.coveragePackages, required int? concurrency, required this.shardIndex, required this.totalShards, @@ -873,6 +889,7 @@ class Configuration { coverage: null, coverageLcov: null, branchCoverage: null, + coveragePackages: null, concurrency: null, shardIndex: null, totalShards: null, @@ -975,6 +992,7 @@ class Configuration { coverage: other.coverage ?? coverage, coverageLcov: other.coverageLcov ?? coverageLcov, branchCoverage: other._branchCoverage ?? _branchCoverage, + coveragePackages: other.coveragePackages ?? coveragePackages, concurrency: other._concurrency ?? _concurrency, shardIndex: other.shardIndex ?? shardIndex, totalShards: other.totalShards ?? totalShards, @@ -1032,6 +1050,7 @@ class Configuration { String? coverage, String? coverageLcov, bool? branchCoverage, + List? coveragePackages, int? concurrency, int? shardIndex, int? totalShards, @@ -1082,6 +1101,7 @@ class Configuration { coverage: coverage ?? this.coverage, coverageLcov: coverageLcov ?? this.coverageLcov, branchCoverage: branchCoverage ?? _branchCoverage, + coveragePackages: coveragePackages ?? this.coveragePackages, concurrency: concurrency ?? _concurrency, shardIndex: shardIndex ?? this.shardIndex, totalShards: totalShards ?? this.totalShards, diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart index e248bf244..a2f452656 100644 --- a/pkgs/test_core/lib/src/runner/configuration/args.dart +++ b/pkgs/test_core/lib/src/runner/configuration/args.dart @@ -180,6 +180,15 @@ final ArgParser _parser = 'Must be paired with --coverage or --coverage-path.', negatable: false, ); + parser.addMultiOption( + 'coverage-package', + help: + 'A regular expression matching packages names to include in\n' + 'the coverage report (if coverage is enabled). If unset,\n' + 'matches the current package name.', + valueHelp: 'regexp', + splitCommas: false, + ); parser.addFlag( 'chain-stack-traces', help: @@ -462,6 +471,7 @@ class _Parser { coverage: coverageDir, coverageLcov: coverageLcov, branchCoverage: branchCoverage, + coveragePackages: _parseCoveragePackagesOption(), concurrency: _parseOption('concurrency', int.parse), shardIndex: shardIndex, totalShards: totalShards, @@ -537,6 +547,17 @@ class _Parser { return {reporter: value.substring(sep + 1)}; }); + List? _parseCoveragePackagesOption() { + final value = _ifParsed>('coverage-package'); + try { + return value?.map(RegExp.new).toList(); + } on FormatException catch (e) { + throw FormatException( + '--coverage-package regular expression syntax is invalid. $e', + ); + } + } + /// Runs [parse], and wraps any [FormatException] it throws with additional /// information. T _wrapFormatException( diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index f44c694c8..19d5f9e6d 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart @@ -427,7 +427,7 @@ Future> _gatherCoverage( false, false, false, - {(await currentPackage).name}, + await _filterCoveragePackages(config.coveragePackages), isolateIds: {isolateId!}, branchCoverage: config.branchCoverage, ); @@ -453,3 +453,16 @@ void _setupPauseAfterTests() { return ServiceExtensionResponse.result(jsonEncode({})); }); } + +Future> _filterCoveragePackages( + List? coveragePackages, +) async { + if (coveragePackages == null || coveragePackages.isEmpty) { + return {(await currentPackage).name}; + } else { + return (await currentPackageConfig).packages + .map((package) => package.name) + .where((name) => coveragePackages.any((re) => re.hasMatch(name))) + .toSet(); + } +} diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml index 65f9d41e6..e90d36f22 100644 --- a/pkgs/test_core/pubspec.yaml +++ b/pkgs/test_core/pubspec.yaml @@ -1,5 +1,5 @@ name: test_core -version: 0.6.14 +version: 0.6.15-wip description: A basic library for writing tests and running them on the VM. repository: https://github.com/dart-lang/test/tree/master/pkgs/test_core issue_tracker: https://github.com/dart-lang/test/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Atest