diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 016fbfa2..4ff097e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,8 +118,9 @@ jobs: done exit $failed - # BRIEF §20.2 visual regression against tests/visual/baselines: a screen beyond the tolerance fails, one without - # a baseline only warns (the baselines README says how to add them). + # BRIEF §20.2 visual regression against tests/visual/baselines: a screen beyond the tolerance fails, and so does one + # without a baseline once any baseline is committed; before the first ones the step warns that it compared nothing + # (the baselines README says how to add them). - name: Screenshot diff if: runner.os == 'Windows' && !cancelled() && hashFiles('out/screenshots/*.png') != '' run: dotnet "$TOOLS" screenshots diff out/screenshots --baseline tests/visual/baselines --out out/screenshot-diff diff --git a/src/Titano.Tools/Commands/ScreenshotsCommand.cs b/src/Titano.Tools/Commands/ScreenshotsCommand.cs index a139dc52..3b20cb5b 100644 --- a/src/Titano.Tools/Commands/ScreenshotsCommand.cs +++ b/src/Titano.Tools/Commands/ScreenshotsCommand.cs @@ -9,7 +9,9 @@ namespace Titano.Tools.Commands; /// PNG with the baseline of the same name (tests/visual/baselines unless --baseline): a pixel differs when /// one of its channels moves by more than --tolerance (of 255), and a screen fails when more than --max-diff /// of its pixels differ (a fraction), when its size changed, or when a baseline has no screenshot any more. A screen -/// without a baseline only warns. Every screen with differing pixels gets <name>-diff.png in --out (the +/// without a baseline fails once the baseline folder holds any PNG, so a screen added later cannot stay ungated; before +/// the first baselines are committed nothing is compared, and the run says so in a warning and in its summary instead +/// of passing for a clean diff. Every screen with differing pixels gets <name>-diff.png in --out (the /// baseline faded to grey, the differing pixels magenta) and the run a summary.md there, copied to the job summary /// in GitHub Actions. screenshots accept <dir> copies the screenshots over the baselines. /// @@ -77,9 +79,12 @@ internal sealed class ScreenshotsCommand : ICommand Resized, Unreadable, - /// A screenshot without a baseline: warns only. + /// A screenshot without a baseline while the baseline folder holds none at all: warns only. New, + /// A screenshot without a baseline while other screens have one. + Unbased, + /// A baseline whose screen was not captured. Missing, } @@ -87,10 +92,11 @@ internal sealed class ScreenshotsCommand : ICommand internal sealed record ScreenResult(string Name, Verdict Verdict, long Differing, long Pixels, string? Detail = null) { public double Fraction => Pixels == 0 ? 0 : (double)Differing / Pixels; - public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Missing; + public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Unbased or Verdict.Missing; } - internal sealed record DiffReport(IReadOnlyList Screens) + /// The baseline folder holds at least one PNG: from then on every screen needs its own. + internal sealed record DiffReport(IReadOnlyList Screens, bool Armed) { public bool Fails => Screens.Any(s => s.Fails); } @@ -103,6 +109,7 @@ internal sealed class ScreenshotsCommand : ICommand { var captured = PngNames(options.Directory); var baselines = Directory.Exists(options.Baseline) ? PngNames(options.Baseline) : new SortedSet(StringComparer.OrdinalIgnoreCase); + var armed = baselines.Count > 0; // Diffs of an earlier run must not pass for this run's. if (Directory.Exists(options.Output)) { @@ -115,7 +122,7 @@ internal sealed class ScreenshotsCommand : ICommand { if (!baselines.Contains(name)) { - results.Add(new ScreenResult(name, Verdict.New, 0, 0)); + results.Add(new ScreenResult(name, armed ? Verdict.Unbased : Verdict.New, 0, 0)); continue; } results.Add(CompareFiles(name, Path.Combine(options.Baseline, name), Path.Combine(options.Directory, name), options)); @@ -124,7 +131,7 @@ internal sealed class ScreenshotsCommand : ICommand results.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.Name, y.Name)); foreach (var r in results) output.WriteLine($"{Label(r.Verdict),-11} {r.Name}{Describe(r, options)}"); - var report = new DiffReport(results); + var report = new DiffReport(results, armed); var markdown = Markdown(report, options); File.WriteAllText(Path.Combine(options.Output, "summary.md"), markdown); if (stepSummary is not null) File.AppendAllText(stepSummary, markdown); @@ -132,14 +139,17 @@ internal sealed class ScreenshotsCommand : ICommand if (annotate) { foreach (var r in results.Where(r => r.Fails)) - output.WriteLine($"::error title={Escape("Screenshot " + Label(r.Verdict), property: true)}::{Escape(r.Name + Describe(r, options), property: false)}"); - // One warning for them all: before the first baselines every screen is new, and GitHub shows only ten. + { + var title = r.Verdict == Verdict.Unbased ? "Screenshot without baseline" : "Screenshot " + Label(r.Verdict); + output.WriteLine($"::error title={Escape(title, property: true)}::{Escape(r.Name + Describe(r, options), property: false)}"); + } + // One warning for them all (GitHub shows only ten): before the first baselines every screen is new. var unbased = results.Where(r => r.Verdict == Verdict.New).Select(r => r.Name).ToList(); if (unbased.Count > 0) { var names = string.Join(", ", unbased.Take(5)) + (unbased.Count > 5 ? ", …" : string.Empty); - output.WriteLine($"::warning title={Escape("Screenshots without baseline", property: true)}::" + - Escape($"{unbased.Count} screenshot(s) have no baseline ({names}). tests/visual/baselines/README.md says how to add them.", property: false)); + output.WriteLine($"::warning title={Escape("No screenshot baselines", property: true)}::" + + Escape($"{NotArmed} {unbased.Count} screenshot(s) without a baseline ({names}); tests/visual/baselines/README.md says how to add them.", property: false)); } } return report; @@ -238,6 +248,7 @@ internal sealed class ScreenshotsCommand : ICommand Verdict.Resized => "resized", Verdict.Unreadable => "unreadable", Verdict.New => "new", + Verdict.Unbased => "no baseline", _ => "missing", }; @@ -247,16 +258,20 @@ internal sealed class ScreenshotsCommand : ICommand Verdict.Within or Verdict.Changed => $": {Percent(r.Fraction)} of the pixels differ ({r.Differing.ToString("N0", CultureInfo.InvariantCulture)}; limit {Percent(options.MaxDiff)})", Verdict.Resized or Verdict.Unreadable => $": {r.Detail}", Verdict.New => ": no baseline", + Verdict.Unbased => ": no baseline, while other screens have one (screenshots accept adds it)", _ => ": a baseline without a screenshot", }; + /// What a run before the first baselines must not hide: it compared nothing, so it could not have failed. + private const string NotArmed = "The baseline folder holds no PNG: nothing was compared, and no visual change can fail this step until baselines are committed."; + private static string Percent(double fraction) => (fraction * 100).ToString("0.###", CultureInfo.InvariantCulture) + " %"; private static string Totals(DiffReport report) { int Count(params Verdict[] verdicts) => report.Screens.Count(s => verdicts.Contains(s.Verdict)); return $"{Count(Verdict.Same, Verdict.Within, Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} compared: {Count(Verdict.Same, Verdict.Within)} within tolerance, " + - $"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New)} without a baseline; {Count(Verdict.Missing)} missing."; + $"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New, Verdict.Unbased)} without a baseline; {Count(Verdict.Missing)} missing."; } private static string Markdown(DiffReport report, DiffOptions options) @@ -264,6 +279,7 @@ internal sealed class ScreenshotsCommand : ICommand var md = new StringBuilder(); md.AppendLine("### Screenshot diff").AppendLine(); md.AppendLine(CultureInfo.InvariantCulture, $"Tolerance {options.Tolerance} of 255 per channel, at most {Percent(options.MaxDiff)} of the pixels.").AppendLine(); + if (!report.Armed) md.AppendLine(CultureInfo.InvariantCulture, $"**{NotArmed}**").AppendLine(); var listed = report.Screens.Where(s => s.Verdict != Verdict.Same).ToList(); if (listed.Count > 0) { diff --git a/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs b/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs index fbe24b5d..626d69fd 100644 --- a/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs +++ b/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs @@ -5,7 +5,10 @@ using static Titano.Tools.Commands.ScreenshotsCommand; namespace Titano.Tools.Tests; -/// The screenshot diff fails on changed screens, tolerates noise and only warns about screens without a baseline (BRIEF §20.2). +/// +/// The screenshot diff fails on changed screens and tolerates noise (BRIEF §20.2); a screen without a baseline fails +/// once any baseline exists, and a run before the first baselines says that it compared nothing. +/// public sealed class ScreenshotsCommandTests(TestContext context) { private const int Size = 100; @@ -90,17 +93,47 @@ public sealed class ScreenshotsCommandTests(TestContext context) } [Test] - public void AScreenWithoutABaselineOnlyWarns() + public void BeforeTheFirstBaselinesTheRunWarnsThatItComparedNothing() { var (captured, baseline, output) = Folders(); WriteScreen(captured, "welcome-dark.png"); - using var log = new StringWriter(); - var report = Run(captured, Path.Combine(baseline, "not-there"), output, log); + WriteScreen(captured, "about-dark.png"); + // A README is not a baseline: the folder as committed before the first screens are accepted. + File.WriteAllText(Path.Combine(baseline, "README.md"), "# Screenshot baselines"); + foreach (var folder in new[] { baseline, Path.Combine(baseline, "not-there") }) + { + using var log = new StringWriter(); + var report = Run(captured, folder, output, log); - Assert.False(report.Fails); - Assert.Equal(Verdict.New, report.Screens.Single().Verdict); - Assert.Contains("::warning title=Screenshots without baseline::1 screenshot(s) have no baseline (welcome-dark.png).", log.ToString()); - Assert.Contains("0 compared", log.ToString()); + Assert.False(report.Fails); + Assert.False(report.Armed); + Assert.True(report.Screens.All(s => s.Verdict == Verdict.New)); + Assert.Contains("::warning title=No screenshot baselines::The baseline folder holds no PNG: nothing was compared, and no visual change can fail " + + "this step until baselines are committed. 2 screenshot(s) without a baseline (about-dark.png, welcome-dark.png);", log.ToString()); + Assert.False(log.ToString().Contains("::error", StringComparison.Ordinal)); + Assert.Contains("0 compared", log.ToString()); + Assert.Contains("**The baseline folder holds no PNG: nothing was compared", File.ReadAllText(Path.Combine(output, "summary.md"))); + } + } + + [Test] + public void OnceAnyBaselineExistsAScreenWithoutOneFails() + { + var (captured, baseline, output) = Folders(); + WriteScreen(baseline, "library-dark.png"); + WriteScreen(captured, "library-dark.png"); + // A screen added after the baselines were committed, whose baseline nobody accepted. + WriteScreen(captured, "library-new-dark.png"); + using var log = new StringWriter(); + var report = Run(captured, baseline, output, log); + + Assert.True(report.Armed); + Assert.True(report.Fails); + Assert.SequenceEqual(new[] { Verdict.Same, Verdict.Unbased }, report.Screens.Select(s => s.Verdict)); + Assert.Contains("::error title=Screenshot without baseline::library-new-dark.png: no baseline, while other screens have one (screenshots accept adds it)", log.ToString()); + Assert.False(log.ToString().Contains("::warning", StringComparison.Ordinal), "the not-compared warning belongs to a run before the first baselines"); + Assert.Contains("1 compared: 1 within tolerance, 0 changed; 1 without a baseline; 0 missing.", log.ToString()); + Assert.False(File.ReadAllText(Path.Combine(output, "summary.md")).Contains("holds no PNG", StringComparison.Ordinal)); } [Test] @@ -138,8 +171,11 @@ public sealed class ScreenshotsCommandTests(TestContext context) int Run(params string[] args) => ScreenshotsCommand.Run(args, _ => null, log); Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--tolerance", "8", "--max-diff", "0.001")); - Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); + // a.png is now within the limit, but b.png still has no baseline beside a.png's. + Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); Assert.FileExists(Path.Combine(output, "a-diff.png")); + File.Copy(Path.Combine(captured, "b.png"), Path.Combine(baseline, "b.png")); + Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); Assert.Equal(0, Run("accept", captured, "--baseline", baseline)); Assert.FileExists(Path.Combine(baseline, "b.png")); Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output)); diff --git a/tests/visual/baselines/README.md b/tests/visual/baselines/README.md index 8c20e44b..7ac9013f 100644 --- a/tests/visual/baselines/README.md +++ b/tests/visual/baselines/README.md @@ -11,7 +11,10 @@ dotnet run --project src/Titano.Tools -- screenshots diff out/screenshots --out - A pixel differs when one of its channels moves by more than `--tolerance` (default 8 of 255); a screen fails when more than `--max-diff` of its pixels differ (default 0.0005, that is 0.05 %), when its size changed, or when a baseline here has no screenshot any more. -- A screenshot without a baseline only warns. +- A screenshot without a baseline fails as soon as this folder holds any PNG: a pull request that adds a screen adds its + baselines too. +- While this folder holds no PNG nothing is compared: the step passes, with a warning and a bold line in the job summary + saying that no visual change can fail it. - `out/screenshot-diff` gets `-diff.png` for every screen with differing pixels (the baseline faded to grey, the differences magenta) and `summary.md`; CI uploads them as the `screenshot-diff` artifact and adds the summary to the job page. @@ -40,3 +43,10 @@ git worktree remove ../ci-screenshots `screenshots accept ` copies every PNG of `` here. Commit only the screens whose change is intended, and delete the baseline of a screen that no longer exists. + +## The first baselines + +None is committed yet. Screenshots captured before the capture was made reproducible cannot serve: their times and +dates follow the moment they were taken (a job finished at "12:18 AM", sample sequences dated from that day). The first +baselines are accepted, all at once, from the `ci-screenshots` commit of the first green `windows-latest` run whose +code captures at `SampleData.Now`; from that commit on, every screen is gated.