From 5ef290d4a4225b444a9fd0a202e59d7837162593 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Fri, 30 Oct 2026 11:05:00 +0900 Subject: [PATCH] test(e2e): let a step redact its output with its own patterns Some steps remove several inputs at once, and vt reports whichever mismatch its parallel validation finds first, so the step's "cache miss" line changes from run to run. A new `redact` step key maps regexes to replacements; the harness applies them in key order to that step's output after the global redactions, and an invalid regex fails when the fixtures load. Co-Authored-By: Claude Opus 5.5 --- crates/vt_bin/tests/e2e_snapshots/README.md | 3 +- .../fixtures/step_redact/package.json | 4 +++ .../fixtures/step_redact/snapshots.toml | 19 ++++++++++++ ...tep_redact_rewrites_only_its_own_output.md | 19 ++++++++++++ crates/vt_bin/tests/e2e_snapshots/main.rs | 30 ++++++++++++++++++- crates/vt_bin/tests/e2e_snapshots/redact.rs | 15 +++++++++- 6 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/package.json create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots.toml create mode 100644 crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots/step_redact_rewrites_only_its_own_output.md diff --git a/crates/vt_bin/tests/e2e_snapshots/README.md b/crates/vt_bin/tests/e2e_snapshots/README.md index 962030e6d..6be4e14f9 100644 --- a/crates/vt_bin/tests/e2e_snapshots/README.md +++ b/crates/vt_bin/tests/e2e_snapshots/README.md @@ -36,7 +36,8 @@ steps = [ Notes: - An array step is shorthand for `{ argv = [...] }`. A string step such as `"vt run build"` doesn't parse, and the test binary panics while loading the fixtures, before any case runs. -- A table step takes `argv` (required), `cwd` (relative to the fixture root, defaulting to the case's `cwd`), `comment` (shown under the step's heading in the snapshot), `envs` (`[name, value]` pairs), `interactions`, and `formatted-snapshot` (keep colors and styles in the snapshot). Other keys are an error. +- A table step takes `argv` (required), `cwd` (relative to the fixture root, defaulting to the case's `cwd`), `comment` (shown under the step's heading in the snapshot), `envs` (`[name, value]` pairs), `interactions`, `formatted-snapshot` (keep colors and styles in the snapshot), and `redact`. Other keys are an error. +- `redact` maps regexes to replacements for output that varies between runs, such as `redact = { 'cache miss: .*' = 'cache miss: ' }`. They rewrite the step's output in key order, after the built-in redactions, and a replacement can refer to capture groups as `$1`. TOML literal strings (`'…'`) keep regex backslashes unescaped. An invalid regex, like an unknown key, makes the test binary panic while loading the fixtures. - A case takes `name` (ASCII letters, digits, and `_` only; it also names the snapshot file), `steps`, and optionally `comment`, `cwd`, `cfg` (a Rust cfg expression such as `'target_os = "linux"'`; the case runs only where it matches), and `ignore` (the case runs only with `-- --ignored`). - `vt` and `vtt` are the binaries built for the test. Other programs are looked up in `PATH`, with `packages/tools/node_modules/.bin` (when installed) ahead of the inherited entries. - `write-key` accepts `up`, `down`, `enter`, `escape`, `backspace`, and `ctrl-c`. diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/package.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/package.json new file mode 100644 index 000000000..4f29dc3a6 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/package.json @@ -0,0 +1,4 @@ +{ + "name": "step-redact", + "private": true +} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots.toml new file mode 100644 index 000000000..8d35f0fc9 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots.toml @@ -0,0 +1,19 @@ +[[e2e]] +name = "step_redact_rewrites_only_its_own_output" +comment = """ +A step's `redact` table rewrites regex matches in that step's output and leaves other steps' output alone. +""" +steps = [ + { argv = [ + "vtt", + "print", + "value:", + "8f3a1c", + ], comment = "`redact` replaces the value", redact = { 'value: \w+' = 'value: ' } }, + { argv = [ + "vtt", + "print", + "value:", + "8f3a1c", + ], comment = "no `redact`, so the value stays" }, +] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots/step_redact_rewrites_only_its_own_output.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots/step_redact_rewrites_only_its_own_output.md new file mode 100644 index 000000000..ea62b3a64 --- /dev/null +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/step_redact/snapshots/step_redact_rewrites_only_its_own_output.md @@ -0,0 +1,19 @@ +# step_redact_rewrites_only_its_own_output + +A step's `redact` table rewrites regex matches in that step's output and leaves other steps' output alone. + +## `vtt print value: 8f3a1c` + +`redact` replaces the value + +``` +value: +``` + +## `vtt print value: 8f3a1c` + +no `redact`, so the value stays + +``` +value: 8f3a1c +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/main.rs b/crates/vt_bin/tests/e2e_snapshots/main.rs index a9f454d03..c8a2f4efe 100644 --- a/crates/vt_bin/tests/e2e_snapshots/main.rs +++ b/crates/vt_bin/tests/e2e_snapshots/main.rs @@ -1,6 +1,7 @@ mod redact; use std::{ + collections::BTreeMap, env::{self, join_paths, split_paths}, ffi::OsStr, io::Write, @@ -59,6 +60,10 @@ struct StepConfig { /// assertion. Default `false` keeps the plain-text behaviour. #[serde(default, rename = "formatted-snapshot")] formatted_snapshot: bool, + /// Regex → replacement pairs for output that varies between runs, applied + /// in key order after the global redactions. + #[serde(default, deserialize_with = "deserialize_redactions")] + redact: Vec<(regex::Regex, Str)>, } impl Step { @@ -148,6 +153,13 @@ impl Step { Self::Simple(_) => false, } } + + fn redactions(&self) -> &[(regex::Regex, Str)] { + match self { + Self::Detailed(config) => &config.redact, + Self::Simple(_) => &[], + } + } } #[derive(serde::Deserialize, Debug, Clone)] @@ -221,6 +233,22 @@ impl WriteKey { } } +/// Compiles a step's `redact` table in key order, so an invalid regex fails +/// when the fixtures load. +fn deserialize_redactions<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let table = as serde::Deserialize>::deserialize(deserializer)?; + table + .into_iter() + .map(|(pattern, replacement)| { + let regex = regex::Regex::new(pattern.as_str()).map_err(serde::de::Error::custom)?; + Ok((regex, replacement)) + }) + .collect() +} + fn deserialize_cfg_expr<'de, D>( deserializer: D, ) -> Result, D::Error> @@ -652,7 +680,7 @@ fn run_case( } } - e2e_outputs.push_str(&redact_e2e_output(output, e2e_stage_path_str)); + e2e_outputs.push_str(&redact_e2e_output(output, e2e_stage_path_str, step.redactions())); // Skip remaining steps if timed out if matches!(termination_state, TerminationState::TimedOut) { diff --git a/crates/vt_bin/tests/e2e_snapshots/redact.rs b/crates/vt_bin/tests/e2e_snapshots/redact.rs index 937cfcc74..89c46ee3f 100644 --- a/crates/vt_bin/tests/e2e_snapshots/redact.rs +++ b/crates/vt_bin/tests/e2e_snapshots/redact.rs @@ -1,5 +1,7 @@ use std::borrow::Cow; +use vt_str::Str; + #[expect( clippy::disallowed_types, reason = "String mutation required by regex replace and cow_replace APIs" @@ -26,7 +28,11 @@ fn redact_string(s: &mut String, redactions: &[(&str, &str)]) { clippy::disallowed_types, reason = "String required by regex replace_all and cow_replace APIs; Path required for CARGO_MANIFEST_DIR path manipulation" )] -pub fn redact_e2e_output(mut output: String, workspace_root: &str) -> String { +pub fn redact_e2e_output( + mut output: String, + workspace_root: &str, + step_redactions: &[(regex::Regex, Str)], +) -> String { let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); // On Windows, canonicalize() may produce verbatim paths (\\?\C:\...) while @@ -124,6 +130,13 @@ pub fn redact_e2e_output(mut output: String, workspace_root: &str) -> String { let mise_warning_regex = regex::Regex::new(r"(?m)^mise WARN\s+.*\n?").unwrap(); output = mise_warning_regex.replace_all(&output, "").into_owned(); + // Apply the step's own `redact` patterns after the global redactions, so + // they match the text the snapshot would otherwise show, and before + // sorting, so diagnostic blocks they normalize sort the same every run. + for (regex, replacement) in step_redactions { + output = regex.replace_all(&output, replacement.as_str()).into_owned(); + } + // Sort consecutive diagnostic blocks to handle non-deterministic tool output // (e.g., oxlint reports warnings in arbitrary order due to multi-threading). // Each block starts with " ! " and ends at the next empty line.