test_runner: add --test-skip-pattern cli option

PR-URL: https://github.com/nodejs/node/pull/52529
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Chemi Atlow <chemi@atlow.co.il>

doc/api/cli.md

@@ -1982,6 +1982,9 @@ A regular expression that configures the test runner to only execute tests
 whose name matches the provided pattern. See the documentation on
 [filtering tests by name][] for more details.
 
+If both `--test-name-pattern` and `--test-skip-pattern` are supplied,
+tests must satisfy **both** requirements in order to be executed.
+
 ### `--test-only`
 
 <!-- YAML
@@ -2050,6 +2053,20 @@ node --test --test-shard=2/3
 node --test --test-shard=3/3
 ```
 
+### `--test-skip-pattern`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+A regular expression that configures the test runner to skip tests
+whose name matches the provided pattern. See the documentation on
+[filtering tests by name][] for more details.
+
+If both `--test-name-pattern` and `--test-skip-pattern` are supplied,
+tests must satisfy **both** requirements in order to be executed.
+
 ### `--test-timeout`
 
 <!-- YAML

doc/api/test.md

@@ -298,12 +298,15 @@ describe.only('a suite', () => {
 
 ## Filtering tests by name
 
-The [`--test-name-pattern`][] command-line option can be used to only run tests
+The [`--test-name-pattern`][] command-line option can be used to only run
-whose name matches the provided pattern. Test name patterns are interpreted as
+tests whose name matches the provided pattern, and the
-JavaScript regular expressions. The `--test-name-pattern` option can be
+[`--test-skip-pattern`][] option can be used to skip tests whose name
-specified multiple times in order to run nested tests. For each test that is
+matches the provided pattern. Test name patterns are interpreted as
-executed, any corresponding test hooks, such as `beforeEach()`, are also
+JavaScript regular expressions. The `--test-name-pattern` and
-run. Tests that are not executed are omitted from the test runner output.
+`--test-skip-pattern` options can be specified multiple times in order to run
+nested tests. For each test that is executed, any corresponding test hooks,
+such as `beforeEach()`, are also run. Tests that are not executed are omitted
+from the test runner output.
 
 Given the following test file, starting Node.js with the
 `--test-name-pattern="test [1-3]"` option would cause the test runner to execute
@@ -327,8 +330,8 @@ test('Test 4', async (t) => {
 
 Test name patterns can also be specified using regular expression literals. This
 allows regular expression flags to be used. In the previous example, starting
-Node.js with `--test-name-pattern="/test [4-5]/i"` would match `Test 4` and
+Node.js with `--test-name-pattern="/test [4-5]/i"` (or `--test-skip-pattern="/test [4-5]/i"`)
-`Test 5` because the pattern is case-insensitive.
+would match `Test 4` and `Test 5` because the pattern is case-insensitive.
 
 To match a single test with a pattern, you can prefix it with all its ancestor
 test names separated by space, to ensure it is unique.
@@ -349,6 +352,9 @@ only `some test` in `test 1`.
 
 Test name patterns do not change the set of files that the test runner executes.
 
+If both `--test-name-pattern` and `--test-skip-pattern` are supplied,
+tests must satisfy **both** requirements in order to be executed.
+
 ## Extraneous asynchronous activity
 
 Once a test function finishes executing, the results are reported as quickly
@@ -3153,6 +3159,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`--test-only`]: cli.md#--test-only
 [`--test-reporter-destination`]: cli.md#--test-reporter-destination
 [`--test-reporter`]: cli.md#--test-reporter
+[`--test-skip-pattern`]: cli.md#--test-skip-pattern
 [`--test`]: cli.md#--test
 [`MockFunctionContext`]: #class-mockfunctioncontext
 [`MockTimers`]: #class-mocktimers

doc/node.1

@@ -442,7 +442,11 @@ option set.
 .
 .It Fl -test-shard
 Test suite shard to execute in a format of <index>/<total>.
-
+.
+.It Fl -test-skip-pattern
+A regular expression that configures the test runner to skip tests
+whose name matches the provided pattern.
+.
 .It Fl -test-timeout
 A number of milliseconds the test execution will fail after.
 .

lib/internal/test_runner/runner.js

@@ -113,7 +113,7 @@ function filterExecArgv(arg, i, arr) {
   !ArrayPrototypeSome(kFilterArgValues, (p) => arg === p || (i > 0 && arr[i - 1] === p) || StringPrototypeStartsWith(arg, `${p}=`));
 }
 
-function getRunArgs(path, { forceExit, inspectPort, testNamePatterns, only }) {
+function getRunArgs(path, { forceExit, inspectPort, testNamePatterns, testSkipPatterns, only }) {
   const argv = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
   if (forceExit === true) {
     ArrayPrototypePush(argv, '--test-force-exit');
@@ -124,6 +124,9 @@ function getRunArgs(path, { forceExit, inspectPort, testNamePatterns, only }) {
   if (testNamePatterns != null) {
     ArrayPrototypeForEach(testNamePatterns, (pattern) => ArrayPrototypePush(argv, `--test-name-pattern=${pattern}`));
   }
+  if (testSkipPatterns != null) {
+    ArrayPrototypeForEach(testSkipPatterns, (pattern) => ArrayPrototypePush(argv, `--test-skip-pattern=${pattern}`));
+  }
   if (only === true) {
     ArrayPrototypePush(argv, '--test-only');
   }
@@ -448,7 +451,7 @@ function watchFiles(testFiles, opts) {
 function run(options = kEmptyObject) {
   validateObject(options, 'options');
 
-  let { testNamePatterns, shard } = options;
+  let { testNamePatterns, testSkipPatterns, shard } = options;
   const {
     concurrency,
     timeout,
@@ -514,6 +517,22 @@ function run(options = kEmptyObject) {
       throw new ERR_INVALID_ARG_TYPE(name, ['string', 'RegExp'], value);
     });
   }
+  if (testSkipPatterns != null) {
+    if (!ArrayIsArray(testSkipPatterns)) {
+      testSkipPatterns = [testSkipPatterns];
+    }
+
+    testSkipPatterns = ArrayPrototypeMap(testSkipPatterns, (value, i) => {
+      if (isRegExp(value)) {
+        return value;
+      }
+      const name = `options.testSkipPatterns[${i}]`;
+      if (typeof value === 'string') {
+        return convertStringToRegExp(value, name);
+      }
+      throw new ERR_INVALID_ARG_TYPE(name, ['string', 'RegExp'], value);
+    });
+  }
 
   const root = createTestTree({ __proto__: null, concurrency, timeout, signal });
   root.harness.shouldColorizeTestFiles ||= shouldColorizeTestFiles(root);
@@ -537,6 +556,7 @@ function run(options = kEmptyObject) {
     signal,
     inspectPort,
     testNamePatterns,
+    testSkipPatterns,
     only,
     forceExit,
   };

lib/internal/test_runner/test.js

@@ -84,6 +84,7 @@ const {
   forceExit,
   sourceMaps,
   testNamePatterns,
+  testSkipPatterns,
   testOnlyFlag,
 } = parseCommandLine();
 let kResistStopPropagation;
@@ -137,6 +138,19 @@ function stopTest(timeout, signal) {
   return deferred.promise;
 }
 
+function testMatchesPattern(test, patterns) {
+  const matchesByNameOrParent = ArrayPrototypeSome(patterns, (re) =>
+    RegExpPrototypeExec(re, test.name) !== null,
+  ) || (test.parent && testMatchesPattern(test.parent, patterns));
+  if (matchesByNameOrParent) return true;
+
+  const testNameWithAncestors = StringPrototypeTrim(test.getTestNameWithAncestors());
+
+  return ArrayPrototypeSome(patterns, (re) =>
+    RegExpPrototypeExec(re, testNameWithAncestors) !== null,
+  );
+}
+
 class TestContext {
   #test;
 
@@ -300,8 +314,7 @@ class Test extends AsyncResource {
         ownAfterEachCount: 0,
       };
 
-      if ((testNamePatterns !== null && !this.matchesTestNamePatterns()) ||
+      if (this.willBeFiltered()) {
-          (testOnlyFlag && !this.only)) {
         this.filtered = true;
         this.parent.filteredSubtestCount++;
       }
@@ -408,18 +421,16 @@ class Test extends AsyncResource {
     }
   }
 
-  matchesTestNamePatterns() {
+  willBeFiltered() {
-    const matchesByNameOrParent = ArrayPrototypeSome(testNamePatterns, (re) =>
+    if (testOnlyFlag && !this.only) return true;
-      RegExpPrototypeExec(re, this.name) !== null,
-    ) ||
-    this.parent?.matchesTestNamePatterns();
 
-    if (matchesByNameOrParent) return true;
+    if (testNamePatterns && !testMatchesPattern(this, testNamePatterns)) {
-
+      return true;
-    const testNameWithAncestors = StringPrototypeTrim(this.getTestNameWithAncestors());
+    }
-    if (!testNameWithAncestors) return false;
+    if (testSkipPatterns && testMatchesPattern(this, testSkipPatterns)) {
-
+      return true;
-    return ArrayPrototypeSome(testNamePatterns, (re) => RegExpPrototypeExec(re, testNameWithAncestors) !== null);
+    }
+    return false;
   }
 
   /**
@@ -987,8 +998,8 @@ class TestHook extends Test {
   getRunArgs() {
     return this.#args;
   }
-  matchesTestNamePatterns() {
+  willBeFiltered() {
-    return true;
+    return false;
   }
   postRun() {
     const { error, loc, parentTest: parent } = this;
@@ -1016,7 +1027,7 @@ class Suite extends Test {
   constructor(options) {
     super(options);
 
-    if (testNamePatterns !== null && !options.skip) {
+    if (testNamePatterns !== null && testSkipPatterns !== null && !options.skip) {
       this.fn = options.fn || this.fn;
       this.skipped = false;
     }
@@ -1050,7 +1061,12 @@ class Suite extends Test {
       // tests that it contains - in case of children matching patterns.
       this.filtered = false;
       this.parent.filteredSubtestCount--;
-    } else if (testOnlyFlag && testNamePatterns == null && this.filteredSubtestCount === this.subtests.length) {
+    } else if (
+      testOnlyFlag &&
+      testNamePatterns == null &&
+      testSkipPatterns == null &&
+      this.filteredSubtestCount === this.subtests.length
+    ) {
       // If no subtests are marked as "only", run them all
       this.filteredSubtestCount = 0;
     }

lib/internal/test_runner/utils.js

@@ -200,6 +200,7 @@ function parseCommandLine() {
   let destinations;
   let reporters;
   let testNamePatterns;
+  let testSkipPatterns;
   let testOnlyFlag;
 
   if (isChildProcessV8) {
@@ -240,6 +241,9 @@ function parseCommandLine() {
         testNamePatternFlag,
         (re) => convertStringToRegExp(re, '--test-name-pattern'),
       ) : null;
+    const testSkipPatternFlag = getOptionValue('--test-skip-pattern');
+    testSkipPatterns = testSkipPatternFlag?.length > 0 ?
+      ArrayPrototypeMap(testSkipPatternFlag, (re) => convertStringToRegExp(re, '--test-skip-pattern')) : null;
   }
 
   globalTestOptions = {
@@ -250,6 +254,7 @@ function parseCommandLine() {
     sourceMaps,
     testOnlyFlag,
     testNamePatterns,
+    testSkipPatterns,
     reporters,
     destinations,
   };

src/node_options.cc

@@ -667,6 +667,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             "run test at specific shard",
             &EnvironmentOptions::test_shard,
             kAllowedInEnvvar);
+  AddOption("--test-skip-pattern",
+            "run tests whose name do not match this regular expression",
+            &EnvironmentOptions::test_skip_pattern);
   AddOption("--test-udp-no-try-send", "",  // For testing only.
             &EnvironmentOptions::test_udp_no_try_send);
   AddOption("--throw-deprecation",

src/node_options.h

@@ -177,6 +177,7 @@ class EnvironmentOptions : public Options {
   bool test_only = false;
   bool test_udp_no_try_send = false;
   std::string test_shard;
+  std::vector<std::string> test_skip_pattern;
   bool throw_deprecation = false;
   bool trace_atomics_wait = false;
   bool trace_deprecation = false;

test/fixtures/test-runner/output/name_and_skip_patterns.js

@@ -0,0 +1,10 @@
+// Flags: --test-skip-pattern=disabled --test-name-pattern=enabled
+'use strict';
+const common = require('../../../common');
+const {
+  test,
+} = require('node:test');
+
+test('disabled', common.mustNotCall());
+test('enabled', common.mustCall());
+test('enabled disabled', common.mustNotCall());

test/fixtures/test-runner/output/name_and_skip_patterns.snapshot

@@ -0,0 +1,15 @@
+TAP version 13
+# Subtest: enabled
+ok 1 - enabled
+  ---
+  duration_ms: *
+  ...
+1..1
+# tests 1
+# suites 0
+# pass 1
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms *

test/fixtures/test-runner/output/skip_pattern.js

@@ -0,0 +1,20 @@
+// Flags: --test-skip-pattern=disabled --test-skip-pattern=/no/i
+'use strict';
+const common = require('../../../common');
+const {
+  describe,
+  it,
+  test,
+} = require('node:test');
+
+test('top level test disabled', common.mustNotCall());
+test('top level skipped test disabled', { skip: true }, common.mustNotCall());
+test('top level skipped test enabled', { skip: true }, common.mustNotCall());
+it('top level it enabled', common.mustCall());
+it('top level it disabled', common.mustNotCall());
+it.skip('top level skipped it disabled', common.mustNotCall());
+it.skip('top level skipped it enabled', common.mustNotCall());
+describe('top level describe', common.mustCall());
+describe.skip('top level skipped describe disabled', common.mustNotCall());
+describe.skip('top level skipped describe enabled', common.mustNotCall());
+test('this will NOt call', common.mustNotCall());

test/fixtures/test-runner/output/skip_pattern.snapshot

@@ -0,0 +1,37 @@
+TAP version 13
+# Subtest: top level skipped test enabled
+ok 1 - top level skipped test enabled # SKIP
+  ---
+  duration_ms: *
+  ...
+# Subtest: top level it enabled
+ok 2 - top level it enabled
+  ---
+  duration_ms: *
+  ...
+# Subtest: top level skipped it enabled
+ok 3 - top level skipped it enabled # SKIP
+  ---
+  duration_ms: *
+  ...
+# Subtest: top level describe
+ok 4 - top level describe
+  ---
+  duration_ms: *
+  type: 'suite'
+  ...
+# Subtest: top level skipped describe enabled
+ok 5 - top level skipped describe enabled # SKIP
+  ---
+  duration_ms: *
+  type: 'suite'
+  ...
+1..5
+# tests 3
+# suites 2
+# pass 1
+# fail 0
+# cancelled 0
+# skipped 2
+# todo 0
+# duration_ms *

test/parallel/test-runner-output.mjs

@@ -119,8 +119,10 @@ const tests = [
   process.features.inspector ? { name: 'test-runner/output/lcov_reporter.js', transform: lcovTransform } : false,
   { name: 'test-runner/output/output.js' },
   { name: 'test-runner/output/output_cli.js' },
+  { name: 'test-runner/output/name_and_skip_patterns.js' },
   { name: 'test-runner/output/name_pattern.js' },
   { name: 'test-runner/output/name_pattern_with_only.js' },
+  { name: 'test-runner/output/skip_pattern.js' },
   { name: 'test-runner/output/unfinished-suite-async-error.js' },
   { name: 'test-runner/output/unresolved_promise.js' },
   { name: 'test-runner/output/default_output.js', transform: specTransform, tty: true },