diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index 39b012143..6ad5c4141 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -13,6 +13,7 @@ all tests with OS `'windows'` would previously still run browser tests on windows, but will now skip all tests including browser tests. * Use a DevTools URL instead of a defunct observatory URL. +* Add flag `--shard-by-suite` to control sharding strategy. * Disable throttling in chrome launch arguments. * Allow package_config `3.x.x`. * Require `analyzer: '>=13.0.0 <15.0.0'` diff --git a/pkgs/test/README.md b/pkgs/test/README.md index 9da43d2a7..472ae9025 100644 --- a/pkgs/test/README.md +++ b/pkgs/test/README.md @@ -204,12 +204,31 @@ dart test --total-shards 3 --shard-index 0 path/to/test.dart dart test --total-shards 3 --shard-index 1 path/to/test.dart dart test --total-shards 3 --shard-index 2 path/to/test.dart ``` -Sharding: This refers to the process of splitting up a large test suite into -smaller subsets (called shards) that can be run independently. Sharding is -particularly useful for distributed testing, where multiple machines are used -to run tests simultaneously. By dividing the test suite into smaller subsets, -you can run tests in parallel across multiple machines, which can significantly -reduce the overall testing time. + +By default, sharding is done by individual tests within each suite. +You can shard by entire test suites (files) using the `--shard-by-suite` flag. + +* When sharding by test (default): Distribute individual test cases across + shards. This balances the test load at the test case level. Test cases from + each suite are sliced continuously, which minimizes how often a suite is + split across shards and helps maximize the re-use of suite `setUpAll` and + `tearDownAll` setups. +* When sharding by suite (using `--shard-by-suite`): Distribute entire test suites + across shards. This can be faster for projects with many small test suites as it + avoids loading every suite in every shard. Because test suites are not split, + any `setUpAll` and `tearDownAll` setups within a suite (including shared helper setups imported across suites) run only on the shard executing that suite. + +Sharding is particularly useful for distributed testing, where multiple +machines are used to run tests simultaneously. By dividing the test suite into +smaller subsets, you can run tests in parallel across multiple machines, which +can significantly reduce the overall testing time. + +### Interaction with Test Filters + +The sharding modes interact differently with filters like `--name` or `--tags`: + +* When sharding by test case (default), the partition is calculated *after* applying filters. This guarantees matching test cases are distributed as evenly as possible across all shards. +* When sharding by test suite (using `--shard-by-suite`), suite-level annotations (such as `@TestOn` platform selectors and suite-level `@Tags` at the top of a file) are evaluated before sharding to filter out non-matching suites. However, test filters which apply to individual test cases or groups (such as `--name`) will be reflected in the distribution when sharding by test case, but not when sharding by test suite. Test invocations with an uneven number of matching test cases between test suites can cause uneven workloads across the shards. ### Test concurrency diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index 97f83630e..8ddcbd17c 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart @@ -84,6 +84,7 @@ $_runtimeCompilers (defaults to "$_defaultConcurrency") --total-shards The total number of invocations of the test runner being run. --shard-index The index of this test runner invocation (of --total-shards). + --shard-by-suite Distribute entire test suites (_test.dart files) across shards instead of individual tests. --timeout The default test timeout. For example: 15s, 2x, none (defaults to "30s") --suite-load-timeout The timeout for loading a test suite. Loading the test suite includes compiling the test suite. For example: 15s, 2m, none diff --git a/pkgs/test/test/runner/shard_test.dart b/pkgs/test/test/runner/shard_test.dart index b255da9c8..6d02d1a32 100644 --- a/pkgs/test/test/runner/shard_test.dart +++ b/pkgs/test/test/runner/shard_test.dart @@ -172,7 +172,291 @@ void main() { await test.shouldExit(79); }); - group('reports an error if', () { + test('shards by suite', () async { + await d.file('1_test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("test 1.1", () {}); + test("test 1.2", () {}); + } + ''').create(); + + await d.file('2_test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("test 2.1", () {}); + test("test 2.2", () {}); + } + ''').create(); + + var test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + ]); + expect( + test.stdout, + containsInOrder([ + '+0: ./1_test.dart: test 1.1', + '+1: ./1_test.dart: test 1.2', + '+2: All tests passed!', + ]), + ); + expect(test.stdout, isNot(contains('./2_test.dart'))); + await test.shouldExit(0); + + test = await runTest([ + '.', + '--shard-index=1', + '--total-shards=2', + '--shard-by-suite', + ]); + expect( + test.stdout, + containsInOrder([ + '+0: ./2_test.dart: test 2.1', + '+1: ./2_test.dart: test 2.2', + '+2: All tests passed!', + ]), + ); + expect(test.stdout, isNot(contains('./1_test.dart'))); + await test.shouldExit(0); + }); + + test('round-robin interleaves suites across shards', () async { + await d.file('1_test.dart', ''' + import 'package:test/test.dart'; + void main() { test("test 1", () {}); } + ''').create(); + + await d.file('2_test.dart', ''' + import 'package:test/test.dart'; + void main() { test("test 2", () {}); } + ''').create(); + + await d.file('3_test.dart', ''' + import 'package:test/test.dart'; + void main() { test("test 3", () {}); } + ''').create(); + + await d.file('4_test.dart', ''' + import 'package:test/test.dart'; + void main() { test("test 4", () {}); } + ''').create(); + + // Shard 0 gets index 0 (1_test) and index 2 (3_test) + var test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + ]); + expect( + test.stdout, + containsInOrder([ + '+0: ./1_test.dart: test 1', + '+1: ./3_test.dart: test 3', + '+2: All tests passed!', + ]), + ); + expect(test.stdout, isNot(contains('./2_test.dart'))); + expect(test.stdout, isNot(contains('./4_test.dart'))); + await test.shouldExit(0); + + // Shard 1 gets index 1 (2_test) and index 3 (4_test) + test = await runTest([ + '.', + '--shard-index=1', + '--total-shards=2', + '--shard-by-suite', + ]); + expect( + test.stdout, + containsInOrder([ + '+0: ./2_test.dart: test 2', + '+1: ./4_test.dart: test 4', + '+2: All tests passed!', + ]), + ); + expect(test.stdout, isNot(contains('./1_test.dart'))); + expect(test.stdout, isNot(contains('./3_test.dart'))); + await test.shouldExit(0); + }); + + test('interaction of --shard-by-suite with name filters', () async { + await d.file('1_test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("match", () {}); + } + ''').create(); + + await d.file('2_test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("other", () {}); + } + ''').create(); + + var test = await runTest([ + '.', + '--shard-index=1', + '--total-shards=2', + '--shard-by-suite', + '--name=match', + ]); + expect(test.stdout, emitsThrough('No tests ran.')); + await test.shouldExit(79); + + test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + '--name=match', + ]); + expect( + test.stdout, + containsInOrder(['+0: ./1_test.dart: match', '+1: All tests passed!']), + ); + await test.shouldExit(0); + }); + + test('interaction of --shard-by-suite with suite-level tag filters', () async { + await d.file('1_test.dart', ''' + @Tags(['match']) + import 'package:test/test.dart'; + + void main() { + test("match", () {}); + } + ''').create(); + + await d.file('2_test.dart', ''' + @Tags(['other']) + import 'package:test/test.dart'; + + void main() { + test("other", () {}); + } + ''').create(); + + // 2_test.dart is filtered out before sharding. Only 1_test.dart is sharded. + // Shard 0 gets 1_test.dart. + var test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + '--tags=match', + ]); + expect( + test.stdout, + containsInOrder(['+0: ./1_test.dart: match', '+1: All tests passed!']), + ); + await test.shouldExit(0); + + // Shard 1 gets no files since only 1 file matched pre-sharding filter. + test = await runTest([ + '.', + '--shard-index=1', + '--total-shards=2', + '--shard-by-suite', + '--tags=match', + ]); + expect(test.stdout, emitsThrough('No tests ran.')); + await test.shouldExit(79); + }); + + test('interaction of --shard-by-suite with test-level tag filters', () async { + await d.file('1_test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("match", tags: "match", () {}); + } + ''').create(); + + await d.file('2_test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("other", tags: "other", () {}); + } + ''').create(); + + // Test-level tags are evaluated within loaded suites after sharding. + // Shard 1 gets 2_test.dart, which has no matching tests. + var test = await runTest([ + '.', + '--shard-index=1', + '--total-shards=2', + '--shard-by-suite', + '--tags=match', + ]); + expect(test.stdout, emitsThrough('No tests ran.')); + await test.shouldExit(79); + + // Shard 0 gets 1_test.dart, which runs. + test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + '--tags=match', + ]); + expect( + test.stdout, + containsInOrder(['+0: ./1_test.dart: match', '+1: All tests passed!']), + ); + await test.shouldExit(0); + }); + + test('interaction of default sharding with name filters', () async { + await d.file('test.dart', ''' + import 'package:test/test.dart'; + + void main() { + test("match 1", () {}); + test("match 2", () {}); + test("other 1", () {}); + test("other 2", () {}); + } + ''').create(); + + var test = await runTest([ + 'test.dart', + '--shard-index=0', + '--total-shards=2', + '--name=match', + ]); + expect( + test.stdout, + containsInOrder(['+0: match 1', '+1: All tests passed!']), + ); + expect(test.stdout, isNot(contains('match 2'))); + await test.shouldExit(0); + + test = await runTest([ + 'test.dart', + '--shard-index=1', + '--total-shards=2', + '--name=match', + ]); + expect( + test.stdout, + containsInOrder(['+0: match 2', '+1: All tests passed!']), + ); + expect(test.stdout, isNot(contains('match 1'))); + await test.shouldExit(0); + }); + + group('errors:', () { test('--shard-index is provided alone', () async { var test = await runTest(['--shard-index=1']); expect( @@ -197,7 +481,7 @@ void main() { await test.shouldExit(exit_codes.usage); }); - test('--shard-index is equal to --total-shards', () async { + test('--shard-index is too large', () async { var test = await runTest(['--shard-index=5', '--total-shards=5']); expect( test.stderr, @@ -206,4 +490,85 @@ void main() { await test.shouldExit(exit_codes.usage); }); }); + + group('--shard-by-suite edge cases:', () { + test('less test suites than shards', () async { + await d.file('1_test.dart', ''' + import 'package:test/test.dart'; + void main() { + test("test 1", () {}); + } + ''').create(); + + var test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + ]); + expect(test.stdout, emitsThrough(contains('+0: ./1_test.dart: test 1'))); + await test.shouldExit(0); + + test = await runTest([ + '.', + '--shard-index=1', + '--total-shards=2', + '--shard-by-suite', + ]); + expect(test.stdout, emitsThrough('No tests ran.')); + await test.shouldExit(79); + }); + + test('no test suites found', () async { + var test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + ]); + expect(test.stderr, emitsThrough('No tests were found.')); + await test.shouldExit(exit_codes.noTestsRan); + }); + + group('interaction with @TestOn:', () { + test('sharding by suite respects @TestOn before sharding', () async { + await d.file('vm_test.dart', ''' + @TestOn('vm') + import 'package:test/test.dart'; + void main() { + test("vm", () {}); + } + ''').create(); + + await d.file('browser_test.dart', ''' + @TestOn('browser') + import 'package:test/test.dart'; + void main() { + test("browser", () {}); + } + ''').create(); + + // browser_test.dart is filtered out before sharding when running on VM. + // So only vm_test.dart is sharded. Shard 0 gets vm_test.dart. + var test = await runTest([ + '.', + '--shard-index=0', + '--total-shards=2', + '--shard-by-suite', + ]); + expect(test.stdout, emitsThrough(contains('+0: ./vm_test.dart: vm'))); + await test.shouldExit(0); + + // Shard 1 gets no files. + test = await runTest([ + '.', + '--shard-index=1', + '--total-shards=2', + '--shard-by-suite', + ]); + expect(test.stdout, emitsThrough('No tests ran.')); + await test.shouldExit(79); + }); + }); + }); } diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart index 26dbd06c9..abc32e5af 100644 --- a/pkgs/test/test/utils.dart +++ b/pkgs/test/test/utils.dart @@ -232,6 +232,7 @@ Configuration configuration({ int? concurrency, int? shardIndex, int? totalShards, + bool? shardBySuite, Map>? testSelections, Iterable? foldTraceExcept, Iterable? foldTraceOnly, @@ -286,6 +287,7 @@ Configuration configuration({ concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, + shardBySuite: shardBySuite, testSelections: testSelections, foldTraceExcept: foldTraceExcept, foldTraceOnly: foldTraceOnly, diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 63428a25c..e80f63d3a 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md @@ -3,12 +3,14 @@ * Add support for `-c cli` (the native CLI compiler) to the vm platform. * Support using the OS platform selector to configure browser tests. * Use a DevTools URL instead of a defunct observatory URL. +* Add flag `--shard-by-suite` to control sharding strategy. * Allow package_config `3.x.x`. * Require `analyzer: '>=13.0.0 <15.0.0'` * Update `parse_metadata.dart` to be compatible with `analyzer >=13.0.0 <15.0.0`. * Use the compact or failures-only reporters by default for tests run directly instead of through the test runner. + ## 0.6.18 * Ignore an error locating the SDK directory on platforms where the diff --git a/pkgs/test_core/lib/src/runner.dart b/pkgs/test_core/lib/src/runner.dart index 455622f81..9af5371a6 100644 --- a/pkgs/test_core/lib/src/runner.dart +++ b/pkgs/test_core/lib/src/runner.dart @@ -7,6 +7,7 @@ import 'dart:io'; import 'package:async/async.dart'; import 'package:boolean_selector/boolean_selector.dart'; +import 'package:path/path.dart' as p; import 'package:stack_trace/stack_trace.dart'; import 'package:test_api/backend.dart' show PlatformSelector, Runtime, SuitePlatform; @@ -269,25 +270,11 @@ class Runner { /// Only tests that match [_config.patterns] will be included in the /// suites once they're loaded. Stream _loadSuites() { - return StreamGroup.merge( - _config.testSelections.entries.map((pathEntry) { - final testPath = pathEntry.key; - final testSelections = pathEntry.value; - final suiteConfig = _config.suiteDefaults.selectTests(testSelections); - if (Directory(testPath).existsSync()) { - return _loader.loadDir(testPath, suiteConfig); - } else if (File(testPath).existsSync()) { - return _loader.loadFile(testPath, suiteConfig); - } else { - return Stream.fromIterable([ - LoadSuite.forLoadException( - LoadException(testPath, 'Does not exist.'), - suiteConfig, - ), - ]); - } - }), - ).map((loadSuite) { + final source = _config.totalShards != null && _config.shardBySuite + ? _shardBySuiteSuites() + : _allSuites(); + + return source.map((loadSuite) { return loadSuite.changeSuite((suite) { _warnForUnknownTags(suite); @@ -406,6 +393,87 @@ class Runner { }); } + /// Returns a stream of all [LoadSuite]s. + Stream _allSuites() { + return StreamGroup.merge( + _config.testSelections.entries.map((pathEntry) { + final testPath = pathEntry.key; + final testSelections = pathEntry.value; + final suiteConfig = _config.suiteDefaults.selectTests(testSelections); + if (Directory(testPath).existsSync()) { + return _loader.loadDir(testPath, suiteConfig); + } else if (File(testPath).existsSync()) { + return _loader.loadFile(testPath, suiteConfig); + } else { + return Stream.fromIterable([ + LoadSuite.forLoadException( + LoadException(testPath, 'Does not exist.'), + suiteConfig, + ), + ]); + } + }), + ); + } + + /// Returns a stream of [LoadSuite]s sharded by suite. + Stream _shardBySuiteSuites() { + final allFiles = []; + for (var testPath in _config.testSelections.keys) { + if (Directory(testPath).existsSync()) { + allFiles.addAll( + Directory(testPath) + .listSync(recursive: true) + .whereType() + .map((f) => f.path) + .where((path) => _config.filename.matches(p.basename(path))), + ); + } else { + // If it's a file or doesn't exist, we add it to the list to be + // sharded and potentially reported as an error later. + allFiles.add(testPath); + } + } + + allFiles.sort(); + + final matchingFiles = allFiles.where((path) { + final entry = _config.testSelections.entries.firstWhere( + (e) => path == e.key || p.isWithin(e.key, path), + orElse: () => _config.testSelections.entries.first, + ); + final suiteConfig = _config.suiteDefaults.selectTests(entry.value); + return _loader.matchesSuite(path, suiteConfig); + }).toList(); + + final shardFiles = []; + for (var i = 0; i < matchingFiles.length; i++) { + if (i % _config.totalShards! == _config.shardIndex!) { + shardFiles.add(matchingFiles[i]); + } + } + + return StreamGroup.merge( + shardFiles.map((path) { + final entry = _config.testSelections.entries.firstWhere( + (e) => path == e.key || p.isWithin(e.key, path), + orElse: () => _config.testSelections.entries.first, + ); + final suiteConfig = _config.suiteDefaults.selectTests(entry.value); + + if (!File(path).existsSync() && !Directory(path).existsSync()) { + return Stream.fromIterable([ + LoadSuite.forLoadException( + LoadException(path, 'Does not exist.'), + suiteConfig, + ), + ]); + } + return _loader.loadFile(path, suiteConfig); + }), + ); + } + /// Prints a warning for any unknown tags referenced in [suite] or its /// children. void _warnForUnknownTags(Suite suite) { @@ -482,15 +550,15 @@ class Runner { return 'the suite itself'; } - /// If sharding is enabled, filters [suite] to only include the tests that - /// should be run in this shard. + /// If sharding by "test" is enabled, filters [suite] to only include the + /// tests that should be run in this shard. /// /// We just take a slice of the tests in each suite corresponding to the shard - /// index. This makes the tests pretty tests across shards, and since the + /// index. This makes the tests pretty even across shards, and since the /// tests are continuous, makes us more likely to be able to re-use /// `setUpAll()` logic. RunnerSuite _shardSuite(RunnerSuite suite) { - if (_config.totalShards == null) return suite; + if (_config.totalShards == null || _config.shardBySuite) return suite; var shardSize = suite.group.testCount / _config.totalShards!; var shardIndex = _config.shardIndex!; diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart index 3f432ada7..95841d55e 100644 --- a/pkgs/test_core/lib/src/runner/configuration.dart +++ b/pkgs/test_core/lib/src/runner/configuration.dart @@ -130,6 +130,10 @@ class Configuration { /// See [shardIndex] for details. final int? totalShards; + /// Whether to distribute tests across shards by suite. + bool get shardBySuite => _shardBySuite ?? false; + final bool? _shardBySuite; + /// The list of packages to fold when producing [StackTrace]s. Set get foldTraceExcept => _foldTraceExcept ?? {}; final Set? _foldTraceExcept; @@ -283,6 +287,7 @@ class Configuration { required int? concurrency, required int? shardIndex, required int? totalShards, + required bool? shardBySuite, required Map>? testSelections, required Iterable? foldTraceExcept, required Iterable? foldTraceOnly, @@ -340,6 +345,7 @@ class Configuration { concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, + shardBySuite: shardBySuite, testSelections: testSelections, foldTraceExcept: foldTraceExcept, foldTraceOnly: foldTraceOnly, @@ -403,6 +409,7 @@ class Configuration { int? concurrency, int? shardIndex, int? totalShards, + bool? shardBySuite, Map>? testSelections, Iterable? foldTraceExcept, Iterable? foldTraceOnly, @@ -458,6 +465,7 @@ class Configuration { concurrency: concurrency, shardIndex: shardIndex, totalShards: totalShards, + shardBySuite: shardBySuite, testSelections: testSelections, foldTraceExcept: foldTraceExcept, foldTraceOnly: foldTraceOnly, @@ -531,6 +539,7 @@ class Configuration { concurrency: null, shardIndex: null, totalShards: null, + shardBySuite: null, testSelections: null, filename: null, chosenPresets: null, @@ -599,6 +608,7 @@ class Configuration { concurrency: null, shardIndex: null, totalShards: null, + shardBySuite: null, testSelections: null, foldTraceExcept: null, foldTraceOnly: null, @@ -669,6 +679,7 @@ class Configuration { coveragePackages: null, shardIndex: null, totalShards: null, + shardBySuite: null, testSelections: null, foldTraceExcept: null, foldTraceOnly: null, @@ -735,6 +746,7 @@ class Configuration { concurrency: null, shardIndex: null, totalShards: null, + shardBySuite: null, foldTraceExcept: null, foldTraceOnly: null, chosenPresets: null, @@ -806,6 +818,7 @@ class Configuration { required int? concurrency, required this.shardIndex, required this.totalShards, + required bool? shardBySuite, required Map>? testSelections, required Iterable? foldTraceExcept, required Iterable? foldTraceOnly, @@ -821,7 +834,8 @@ class Configuration { required BooleanSelector? excludeTags, required Iterable? globalPatterns, required SuiteConfiguration? suiteDefaults, - }) : _help = help, + }) : _shardBySuite = shardBySuite, + _help = help, _version = version, _pauseAfterLoad = pauseAfterLoad, _debug = debug, @@ -899,6 +913,7 @@ class Configuration { concurrency: null, shardIndex: null, totalShards: null, + shardBySuite: null, testSelections: null, foldTraceExcept: null, foldTraceOnly: null, @@ -1002,6 +1017,7 @@ class Configuration { concurrency: other._concurrency ?? _concurrency, shardIndex: other.shardIndex ?? shardIndex, totalShards: other.totalShards ?? totalShards, + shardBySuite: other._shardBySuite ?? _shardBySuite, testSelections: other._testSelections ?? _testSelections, foldTraceExcept: foldTraceExcept, foldTraceOnly: foldTraceOnly, @@ -1059,6 +1075,7 @@ class Configuration { int? concurrency, int? shardIndex, int? totalShards, + bool? shardBySuite, Map>? testSelections, Iterable? exceptPackages, Iterable? onlyPackages, @@ -1110,6 +1127,7 @@ class Configuration { concurrency: concurrency ?? _concurrency, shardIndex: shardIndex ?? this.shardIndex, totalShards: totalShards ?? this.totalShards, + shardBySuite: shardBySuite ?? _shardBySuite, testSelections: testSelections ?? _testSelections, foldTraceExcept: exceptPackages ?? _foldTraceExcept, foldTraceOnly: onlyPackages ?? _foldTraceOnly, diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart index 1714784f3..e7799c9ef 100644 --- a/pkgs/test_core/lib/src/runner/configuration/args.dart +++ b/pkgs/test_core/lib/src/runner/configuration/args.dart @@ -129,6 +129,12 @@ final ArgParser _parser = (() { 'shard-index', help: 'The index of this test runner invocation (of --total-shards).', ); + parser.addFlag( + 'shard-by-suite', + help: + 'Distribute entire test suites (_test.dart files) across shards instead of individual tests.', + negatable: false, + ); parser.addOption( 'pub-serve', help: '[Removed] The port of a pub serve instance serving "test/".', @@ -379,6 +385,7 @@ class _Parser { var shardIndex = _parseOption('shard-index', int.parse); var totalShards = _parseOption('total-shards', int.parse); + var shardBySuite = _ifParsed('shard-by-suite') as bool?; if ((shardIndex == null) != (totalShards == null)) { throw const FormatException( '--shard-index and --total-shards may only be passed together.', @@ -477,6 +484,7 @@ class _Parser { concurrency: _parseOption('concurrency', int.parse), shardIndex: shardIndex, totalShards: totalShards, + shardBySuite: shardBySuite, timeout: _parseOption('timeout', Timeout.parse), suiteLoadTimeout: _parseOption('suite-load-timeout', Timeout.parse), globalPatterns: patterns, diff --git a/pkgs/test_core/lib/src/runner/loader.dart b/pkgs/test_core/lib/src/runner/loader.dart index ac238c549..3a42c8816 100644 --- a/pkgs/test_core/lib/src/runner/loader.dart +++ b/pkgs/test_core/lib/src/runner/loader.dart @@ -6,10 +6,12 @@ import 'dart:async'; import 'dart:io'; import 'package:async/async.dart'; +import 'package:boolean_selector/boolean_selector.dart'; import 'package:path/path.dart' as p; import 'package:source_span/source_span.dart'; import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports import 'package:yaml/yaml.dart'; @@ -29,6 +31,84 @@ import 'vm/platform.dart'; /// A class for finding test files and loading them into a runnable form. class Loader { + /// Cache of metadata parsed from test files. + final _metadataCache = {}; + + /// Parses and returns the suite metadata for [path], caching the result. + Metadata parseSuiteMetadata(String path) { + final cached = _metadataCache.putIfAbsent(path, () { + if (!File(path).existsSync()) return Metadata.empty; + try { + return parseMetadata( + path, + File(path).readAsStringSync(), + _runtimeVariables.toSet(), + ); + } catch (e) { + return e; + } + }); + if (cached is Metadata) return cached; + throw cached; + } + + /// Returns whether the test suite at [path] matches target platform and tag filters + /// specified in [suiteConfig]. + bool matchesSuite(String path, SuiteConfiguration suiteConfig) { + if (!File(path).existsSync()) return true; + Metadata metadata; + try { + metadata = parseSuiteMetadata(path); + } catch (_) { + return true; + } + + final mergedConfig = suiteConfig.merge( + SuiteConfiguration.fromMetadata(metadata), + ); + + if (_config.excludeTags.evaluate(mergedConfig.metadata.tags.contains)) { + return false; + } + + if (_config.includeTags != BooleanSelector.all && + mergedConfig.metadata.tags.isNotEmpty) { + if (!_config.includeTags.evaluate(mergedConfig.metadata.tags.contains)) { + return false; + } + } + + var hasMatchingPlatform = false; + for (var runtimeName in mergedConfig.runtimes) { + var runtime = findRuntime(runtimeName); + if (runtime == null) continue; + final compilers = { + for (var selection + in mergedConfig.compilerSelections ?? []) + if (runtime.supportedCompilers.contains(selection.compiler) && + (selection.platformSelector == null || + selection.platformSelector!.evaluate( + currentPlatform(runtime, selection.compiler), + ))) + selection.compiler, + }; + if (compilers.isEmpty) compilers.add(runtime.defaultCompiler); + + for (var compiler in compilers) { + var platform = currentPlatform(runtime, compiler); + if (mergedConfig.metadata.testOn.evaluate(platform)) { + hasMatchingPlatform = true; + break; + } + } + if (hasMatchingPlatform) break; + } + + if (!hasMatchingPlatform) return false; + + return true; + } + /// The test runner configuration. final _config = Configuration.current; @@ -173,13 +253,7 @@ class Loader { ) async* { try { suiteConfig = suiteConfig.merge( - SuiteConfiguration.fromMetadata( - parseMetadata( - path, - File(path).readAsStringSync(), - _runtimeVariables.toSet(), - ), - ), + SuiteConfiguration.fromMetadata(parseSuiteMetadata(path)), ); } on ArgumentError catch (_) { // Ignore the analyzer's error, since its formatting is much worse than