From 2c643fba96268cc44a29006ecceaf6eace9e93e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 10:34:25 +0000 Subject: [PATCH] fix(tools): screenshot diff fails a screen without a baseline once any is committed A screenshot without a baseline only warned, so with no baseline committed every screen was "new" and the CI step could not fail, and a screen added after the first baselines could stay ungated for good. Now the baseline folder is armed as soon as it holds a PNG: from then on a screen without its own baseline fails (Verdict.Unbased, an error annotation). Before the first baselines the run says plainly, in its warning and in bold in the job summary, that it compared nothing and that no visual change can fail it. The baselines README explains why the existing ci-screenshots pictures cannot serve (captured before the clock was fixed: times and dates follow the capture moment) and how the first set is accepted from the first green windows-latest run of the reproducible capture. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .github/workflows/ci.yml | 5 +- .../Commands/ScreenshotsCommand.cs | 38 +++++++++---- .../ScreenshotsCommandTests.cs | 54 +++++++++++++++---- tests/visual/baselines/README.md | 12 ++++- 4 files changed, 86 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 016fbfa2..4ff097e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,8 +118,9 @@ jobs: done exit $failed - # BRIEF §20.2 visual regression against tests/visual/baselines: a screen beyond the tolerance fails, one without - # a baseline only warns (the baselines README says how to add them). + # BRIEF §20.2 visual regression against tests/visual/baselines: a screen beyond the tolerance fails, and so does one + # without a baseline once any baseline is committed; before the first ones the step warns that it compared nothing + # (the baselines README says how to add them). - name: Screenshot diff if: runner.os == 'Windows' && !cancelled() && hashFiles('out/screenshots/*.png') != '' run: dotnet "$TOOLS" screenshots diff out/screenshots --baseline tests/visual/baselines --out out/screenshot-diff diff --git a/src/Titano.Tools/Commands/ScreenshotsCommand.cs b/src/Titano.Tools/Commands/ScreenshotsCommand.cs index a139dc52..3b20cb5b 100644 --- a/src/Titano.Tools/Commands/ScreenshotsCommand.cs +++ b/src/Titano.Tools/Commands/ScreenshotsCommand.cs @@ -9,7 +9,9 @@ namespace Titano.Tools.Commands; /// PNG with the baseline of the same name (tests/visual/baselines unless --baseline): a pixel differs when /// one of its channels moves by more than --tolerance (of 255), and a screen fails when more than --max-diff /// of its pixels differ (a fraction), when its size changed, or when a baseline has no screenshot any more. A screen -/// without a baseline only warns. Every screen with differing pixels gets <name>-diff.png in --out (the +/// without a baseline fails once the baseline folder holds any PNG, so a screen added later cannot stay ungated; before +/// the first baselines are committed nothing is compared, and the run says so in a warning and in its summary instead +/// of passing for a clean diff. Every screen with differing pixels gets <name>-diff.png in --out (the /// baseline faded to grey, the differing pixels magenta) and the run a summary.md there, copied to the job summary /// in GitHub Actions. screenshots accept <dir> copies the screenshots over the baselines. /// @@ -77,9 +79,12 @@ internal sealed class ScreenshotsCommand : ICommand Resized, Unreadable, - /// A screenshot without a baseline: warns only. + /// A screenshot without a baseline while the baseline folder holds none at all: warns only. New, + /// A screenshot without a baseline while other screens have one. + Unbased, + /// A baseline whose screen was not captured. Missing, } @@ -87,10 +92,11 @@ internal sealed class ScreenshotsCommand : ICommand internal sealed record ScreenResult(string Name, Verdict Verdict, long Differing, long Pixels, string? Detail = null) { public double Fraction => Pixels == 0 ? 0 : (double)Differing / Pixels; - public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Missing; + public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Unbased or Verdict.Missing; } - internal sealed record DiffReport(IReadOnlyList Screens) + /// The baseline folder holds at least one PNG: from then on every screen needs its own. + internal sealed record DiffReport(IReadOnlyList Screens, bool Armed) { public bool Fails => Screens.Any(s => s.Fails); } @@ -103,6 +109,7 @@ internal sealed class ScreenshotsCommand : ICommand { var captured = PngNames(options.Directory); var baselines = Directory.Exists(options.Baseline) ? PngNames(options.Baseline) : new SortedSet(StringComparer.OrdinalIgnoreCase); + var armed = baselines.Count > 0; // Diffs of an earlier run must not pass for this run's. if (Directory.Exists(options.Output)) { @@ -115,7 +122,7 @@ internal sealed class ScreenshotsCommand : ICommand { if (!baselines.Contains(name)) { - results.Add(new ScreenResult(name, Verdict.New, 0, 0)); + results.Add(new ScreenResult(name, armed ? Verdict.Unbased : Verdict.New, 0, 0)); continue; } results.Add(CompareFiles(name, Path.Combine(options.Baseline, name), Path.Combine(options.Directory, name), options)); @@ -124,7 +131,7 @@ internal sealed class ScreenshotsCommand : ICommand results.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.Name, y.Name)); foreach (var r in results) output.WriteLine($"{Label(r.Verdict),-11} {r.Name}{Describe(r, options)}"); - var report = new DiffReport(results); + var report = new DiffReport(results, armed); var markdown = Markdown(report, options); File.WriteAllText(Path.Combine(options.Output, "summary.md"), markdown); if (stepSummary is not null) File.AppendAllText(stepSummary, markdown); @@ -132,14 +139,17 @@ internal sealed class ScreenshotsCommand : ICommand if (annotate) { foreach (var r in results.Where(r => r.Fails)) - output.WriteLine($"::error title={Escape("Screenshot " + Label(r.Verdict), property: true)}::{Escape(r.Name + Describe(r, options), property: false)}"); - // One warning for them all: before the first baselines every screen is new, and GitHub shows only ten. + { + var title = r.Verdict == Verdict.Unbased ? "Screenshot without baseline" : "Screenshot " + Label(r.Verdict); + output.WriteLine($"::error title={Escape(title, property: true)}::{Escape(r.Name + Describe(r, options), property: false)}"); + } + // One warning for them all (GitHub shows only ten): before the first baselines every screen is new. var unbased = results.Where(r => r.Verdict == Verdict.New).Select(r => r.Name).ToList(); if (unbased.Count > 0) { var names = string.Join(", ", unbased.Take(5)) + (unbased.Count > 5 ? ", …" : string.Empty); - output.WriteLine($"::warning title={Escape("Screenshots without baseline", property: true)}::" + - Escape($"{unbased.Count} screenshot(s) have no baseline ({names}). tests/visual/baselines/README.md says how to add them.", property: false)); + output.WriteLine($"::warning title={Escape("No screenshot baselines", property: true)}::" + + Escape($"{NotArmed} {unbased.Count} screenshot(s) without a baseline ({names}); tests/visual/baselines/README.md says how to add them.", property: false)); } } return report; @@ -238,6 +248,7 @@ internal sealed class ScreenshotsCommand : ICommand Verdict.Resized => "resized", Verdict.Unreadable => "unreadable", Verdict.New => "new", + Verdict.Unbased => "no baseline", _ => "missing", }; @@ -247,16 +258,20 @@ internal sealed class ScreenshotsCommand : ICommand Verdict.Within or Verdict.Changed => $": {Percent(r.Fraction)} of the pixels differ ({r.Differing.ToString("N0", CultureInfo.InvariantCulture)}; limit {Percent(options.MaxDiff)})", Verdict.Resized or Verdict.Unreadable => $": {r.Detail}", Verdict.New => ": no baseline", + Verdict.Unbased => ": no baseline, while other screens have one (screenshots accept adds it)", _ => ": a baseline without a screenshot", }; + /// What a run before the first baselines must not hide: it compared nothing, so it could not have failed. + private const string NotArmed = "The baseline folder holds no PNG: nothing was compared, and no visual change can fail this step until baselines are committed."; + private static string Percent(double fraction) => (fraction * 100).ToString("0.###", CultureInfo.InvariantCulture) + " %"; private static string Totals(DiffReport report) { int Count(params Verdict[] verdicts) => report.Screens.Count(s => verdicts.Contains(s.Verdict)); return $"{Count(Verdict.Same, Verdict.Within, Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} compared: {Count(Verdict.Same, Verdict.Within)} within tolerance, " + - $"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New)} without a baseline; {Count(Verdict.Missing)} missing."; + $"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New, Verdict.Unbased)} without a baseline; {Count(Verdict.Missing)} missing."; } private static string Markdown(DiffReport report, DiffOptions options) @@ -264,6 +279,7 @@ internal sealed class ScreenshotsCommand : ICommand var md = new StringBuilder(); md.AppendLine("### Screenshot diff").AppendLine(); md.AppendLine(CultureInfo.InvariantCulture, $"Tolerance {options.Tolerance} of 255 per channel, at most {Percent(options.MaxDiff)} of the pixels.").AppendLine(); + if (!report.Armed) md.AppendLine(CultureInfo.InvariantCulture, $"**{NotArmed}**").AppendLine(); var listed = report.Screens.Where(s => s.Verdict != Verdict.Same).ToList(); if (listed.Count > 0) { diff --git a/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs b/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs index fbe24b5d..626d69fd 100644 --- a/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs +++ b/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs @@ -5,7 +5,10 @@ using static Titano.Tools.Commands.ScreenshotsCommand; namespace Titano.Tools.Tests; -/// The screenshot diff fails on changed screens, tolerates noise and only warns about screens without a baseline (BRIEF §20.2). +/// +/// The screenshot diff fails on changed screens and tolerates noise (BRIEF §20.2); a screen without a baseline fails +/// once any baseline exists, and a run before the first baselines says that it compared nothing. +/// public sealed class ScreenshotsCommandTests(TestContext context) { private const int Size = 100; @@ -90,17 +93,47 @@ public sealed class ScreenshotsCommandTests(TestContext context) } [Test] - public void AScreenWithoutABaselineOnlyWarns() + public void BeforeTheFirstBaselinesTheRunWarnsThatItComparedNothing() { var (captured, baseline, output) = Folders(); WriteScreen(captured, "welcome-dark.png"); - using var log = new StringWriter(); - var report = Run(captured, Path.Combine(baseline, "not-there"), output, log); + WriteScreen(captured, "about-dark.png"); + // A README is not a baseline: the folder as committed before the first screens are accepted. + File.WriteAllText(Path.Combine(baseline, "README.md"), "# Screenshot baselines"); + foreach (var folder in new[] { baseline, Path.Combine(baseline, "not-there") }) + { + using var log = new StringWriter(); + var report = Run(captured, folder, output, log); - Assert.False(report.Fails); - Assert.Equal(Verdict.New, report.Screens.Single().Verdict); - Assert.Contains("::warning title=Screenshots without baseline::1 screenshot(s) have no baseline (welcome-dark.png).", log.ToString()); - Assert.Contains("0 compared", log.ToString()); + Assert.False(report.Fails); + Assert.False(report.Armed); + Assert.True(report.Screens.All(s => s.Verdict == Verdict.New)); + Assert.Contains("::warning title=No screenshot baselines::The baseline folder holds no PNG: nothing was compared, and no visual change can fail " + + "this step until baselines are committed. 2 screenshot(s) without a baseline (about-dark.png, welcome-dark.png);", log.ToString()); + Assert.False(log.ToString().Contains("::error", StringComparison.Ordinal)); + Assert.Contains("0 compared", log.ToString()); + Assert.Contains("**The baseline folder holds no PNG: nothing was compared", File.ReadAllText(Path.Combine(output, "summary.md"))); + } + } + + [Test] + public void OnceAnyBaselineExistsAScreenWithoutOneFails() + { + var (captured, baseline, output) = Folders(); + WriteScreen(baseline, "library-dark.png"); + WriteScreen(captured, "library-dark.png"); + // A screen added after the baselines were committed, whose baseline nobody accepted. + WriteScreen(captured, "library-new-dark.png"); + using var log = new StringWriter(); + var report = Run(captured, baseline, output, log); + + Assert.True(report.Armed); + Assert.True(report.Fails); + Assert.SequenceEqual(new[] { Verdict.Same, Verdict.Unbased }, report.Screens.Select(s => s.Verdict)); + Assert.Contains("::error title=Screenshot without baseline::library-new-dark.png: no baseline, while other screens have one (screenshots accept adds it)", log.ToString()); + Assert.False(log.ToString().Contains("::warning", StringComparison.Ordinal), "the not-compared warning belongs to a run before the first baselines"); + Assert.Contains("1 compared: 1 within tolerance, 0 changed; 1 without a baseline; 0 missing.", log.ToString()); + Assert.False(File.ReadAllText(Path.Combine(output, "summary.md")).Contains("holds no PNG", StringComparison.Ordinal)); } [Test] @@ -138,8 +171,11 @@ public sealed class ScreenshotsCommandTests(TestContext context) int Run(params string[] args) => ScreenshotsCommand.Run(args, _ => null, log); Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--tolerance", "8", "--max-diff", "0.001")); - Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); + // a.png is now within the limit, but b.png still has no baseline beside a.png's. + Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); Assert.FileExists(Path.Combine(output, "a-diff.png")); + File.Copy(Path.Combine(captured, "b.png"), Path.Combine(baseline, "b.png")); + Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); Assert.Equal(0, Run("accept", captured, "--baseline", baseline)); Assert.FileExists(Path.Combine(baseline, "b.png")); Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output)); diff --git a/tests/visual/baselines/README.md b/tests/visual/baselines/README.md index 8c20e44b..7ac9013f 100644 --- a/tests/visual/baselines/README.md +++ b/tests/visual/baselines/README.md @@ -11,7 +11,10 @@ dotnet run --project src/Titano.Tools -- screenshots diff out/screenshots --out - A pixel differs when one of its channels moves by more than `--tolerance` (default 8 of 255); a screen fails when more than `--max-diff` of its pixels differ (default 0.0005, that is 0.05 %), when its size changed, or when a baseline here has no screenshot any more. -- A screenshot without a baseline only warns. +- A screenshot without a baseline fails as soon as this folder holds any PNG: a pull request that adds a screen adds its + baselines too. +- While this folder holds no PNG nothing is compared: the step passes, with a warning and a bold line in the job summary + saying that no visual change can fail it. - `out/screenshot-diff` gets `-diff.png` for every screen with differing pixels (the baseline faded to grey, the differences magenta) and `summary.md`; CI uploads them as the `screenshot-diff` artifact and adds the summary to the job page. @@ -40,3 +43,10 @@ git worktree remove ../ci-screenshots `screenshots accept ` copies every PNG of `` here. Commit only the screens whose change is intended, and delete the baseline of a screen that no longer exists. + +## The first baselines + +None is committed yet. Screenshots captured before the capture was made reproducible cannot serve: their times and +dates follow the moment they were taken (a job finished at "12:18 AM", sample sequences dated from that day). The first +baselines are accepted, all at once, from the `ci-screenshots` commit of the first green `windows-latest` run whose +code captures at `SampleData.Now`; from that commit on, every screen is gated.