diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 016fbfa2..4ff097e7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -118,8 +118,9 @@ jobs:
done
exit $failed
- # BRIEF §20.2 visual regression against tests/visual/baselines: a screen beyond the tolerance fails, one without
- # a baseline only warns (the baselines README says how to add them).
+ # BRIEF §20.2 visual regression against tests/visual/baselines: a screen beyond the tolerance fails, and so does one
+ # without a baseline once any baseline is committed; before the first ones the step warns that it compared nothing
+ # (the baselines README says how to add them).
- name: Screenshot diff
if: runner.os == 'Windows' && !cancelled() && hashFiles('out/screenshots/*.png') != ''
run: dotnet "$TOOLS" screenshots diff out/screenshots --baseline tests/visual/baselines --out out/screenshot-diff
diff --git a/src/Titano.Tools/Commands/ScreenshotsCommand.cs b/src/Titano.Tools/Commands/ScreenshotsCommand.cs
index a139dc52..3b20cb5b 100644
--- a/src/Titano.Tools/Commands/ScreenshotsCommand.cs
+++ b/src/Titano.Tools/Commands/ScreenshotsCommand.cs
@@ -9,7 +9,9 @@ namespace Titano.Tools.Commands;
/// PNG with the baseline of the same name (tests/visual/baselines unless --baseline): a pixel differs when
/// one of its channels moves by more than --tolerance (of 255), and a screen fails when more than --max-diff
/// of its pixels differ (a fraction), when its size changed, or when a baseline has no screenshot any more. A screen
-/// without a baseline only warns. Every screen with differing pixels gets <name>-diff.png in --out (the
+/// without a baseline fails once the baseline folder holds any PNG, so a screen added later cannot stay ungated; before
+/// the first baselines are committed nothing is compared, and the run says so in a warning and in its summary instead
+/// of passing for a clean diff. Every screen with differing pixels gets <name>-diff.png in --out (the
/// baseline faded to grey, the differing pixels magenta) and the run a summary.md there, copied to the job summary
/// in GitHub Actions. screenshots accept <dir> copies the screenshots over the baselines.
///
@@ -77,9 +79,12 @@ internal sealed class ScreenshotsCommand : ICommand
Resized,
Unreadable,
- /// A screenshot without a baseline: warns only.
+ /// A screenshot without a baseline while the baseline folder holds none at all: warns only.
New,
+ /// A screenshot without a baseline while other screens have one.
+ Unbased,
+
/// A baseline whose screen was not captured.
Missing,
}
@@ -87,10 +92,11 @@ internal sealed class ScreenshotsCommand : ICommand
internal sealed record ScreenResult(string Name, Verdict Verdict, long Differing, long Pixels, string? Detail = null)
{
public double Fraction => Pixels == 0 ? 0 : (double)Differing / Pixels;
- public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Missing;
+ public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Unbased or Verdict.Missing;
}
- internal sealed record DiffReport(IReadOnlyList Screens)
+ /// The baseline folder holds at least one PNG: from then on every screen needs its own.
+ internal sealed record DiffReport(IReadOnlyList Screens, bool Armed)
{
public bool Fails => Screens.Any(s => s.Fails);
}
@@ -103,6 +109,7 @@ internal sealed class ScreenshotsCommand : ICommand
{
var captured = PngNames(options.Directory);
var baselines = Directory.Exists(options.Baseline) ? PngNames(options.Baseline) : new SortedSet(StringComparer.OrdinalIgnoreCase);
+ var armed = baselines.Count > 0;
// Diffs of an earlier run must not pass for this run's.
if (Directory.Exists(options.Output))
{
@@ -115,7 +122,7 @@ internal sealed class ScreenshotsCommand : ICommand
{
if (!baselines.Contains(name))
{
- results.Add(new ScreenResult(name, Verdict.New, 0, 0));
+ results.Add(new ScreenResult(name, armed ? Verdict.Unbased : Verdict.New, 0, 0));
continue;
}
results.Add(CompareFiles(name, Path.Combine(options.Baseline, name), Path.Combine(options.Directory, name), options));
@@ -124,7 +131,7 @@ internal sealed class ScreenshotsCommand : ICommand
results.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.Name, y.Name));
foreach (var r in results) output.WriteLine($"{Label(r.Verdict),-11} {r.Name}{Describe(r, options)}");
- var report = new DiffReport(results);
+ var report = new DiffReport(results, armed);
var markdown = Markdown(report, options);
File.WriteAllText(Path.Combine(options.Output, "summary.md"), markdown);
if (stepSummary is not null) File.AppendAllText(stepSummary, markdown);
@@ -132,14 +139,17 @@ internal sealed class ScreenshotsCommand : ICommand
if (annotate)
{
foreach (var r in results.Where(r => r.Fails))
- output.WriteLine($"::error title={Escape("Screenshot " + Label(r.Verdict), property: true)}::{Escape(r.Name + Describe(r, options), property: false)}");
- // One warning for them all: before the first baselines every screen is new, and GitHub shows only ten.
+ {
+ var title = r.Verdict == Verdict.Unbased ? "Screenshot without baseline" : "Screenshot " + Label(r.Verdict);
+ output.WriteLine($"::error title={Escape(title, property: true)}::{Escape(r.Name + Describe(r, options), property: false)}");
+ }
+ // One warning for them all (GitHub shows only ten): before the first baselines every screen is new.
var unbased = results.Where(r => r.Verdict == Verdict.New).Select(r => r.Name).ToList();
if (unbased.Count > 0)
{
var names = string.Join(", ", unbased.Take(5)) + (unbased.Count > 5 ? ", …" : string.Empty);
- output.WriteLine($"::warning title={Escape("Screenshots without baseline", property: true)}::" +
- Escape($"{unbased.Count} screenshot(s) have no baseline ({names}). tests/visual/baselines/README.md says how to add them.", property: false));
+ output.WriteLine($"::warning title={Escape("No screenshot baselines", property: true)}::" +
+ Escape($"{NotArmed} {unbased.Count} screenshot(s) without a baseline ({names}); tests/visual/baselines/README.md says how to add them.", property: false));
}
}
return report;
@@ -238,6 +248,7 @@ internal sealed class ScreenshotsCommand : ICommand
Verdict.Resized => "resized",
Verdict.Unreadable => "unreadable",
Verdict.New => "new",
+ Verdict.Unbased => "no baseline",
_ => "missing",
};
@@ -247,16 +258,20 @@ internal sealed class ScreenshotsCommand : ICommand
Verdict.Within or Verdict.Changed => $": {Percent(r.Fraction)} of the pixels differ ({r.Differing.ToString("N0", CultureInfo.InvariantCulture)}; limit {Percent(options.MaxDiff)})",
Verdict.Resized or Verdict.Unreadable => $": {r.Detail}",
Verdict.New => ": no baseline",
+ Verdict.Unbased => ": no baseline, while other screens have one (screenshots accept adds it)",
_ => ": a baseline without a screenshot",
};
+ /// What a run before the first baselines must not hide: it compared nothing, so it could not have failed.
+ private const string NotArmed = "The baseline folder holds no PNG: nothing was compared, and no visual change can fail this step until baselines are committed.";
+
private static string Percent(double fraction) => (fraction * 100).ToString("0.###", CultureInfo.InvariantCulture) + " %";
private static string Totals(DiffReport report)
{
int Count(params Verdict[] verdicts) => report.Screens.Count(s => verdicts.Contains(s.Verdict));
return $"{Count(Verdict.Same, Verdict.Within, Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} compared: {Count(Verdict.Same, Verdict.Within)} within tolerance, " +
- $"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New)} without a baseline; {Count(Verdict.Missing)} missing.";
+ $"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New, Verdict.Unbased)} without a baseline; {Count(Verdict.Missing)} missing.";
}
private static string Markdown(DiffReport report, DiffOptions options)
@@ -264,6 +279,7 @@ internal sealed class ScreenshotsCommand : ICommand
var md = new StringBuilder();
md.AppendLine("### Screenshot diff").AppendLine();
md.AppendLine(CultureInfo.InvariantCulture, $"Tolerance {options.Tolerance} of 255 per channel, at most {Percent(options.MaxDiff)} of the pixels.").AppendLine();
+ if (!report.Armed) md.AppendLine(CultureInfo.InvariantCulture, $"**{NotArmed}**").AppendLine();
var listed = report.Screens.Where(s => s.Verdict != Verdict.Same).ToList();
if (listed.Count > 0)
{
diff --git a/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs b/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs
index fbe24b5d..626d69fd 100644
--- a/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs
+++ b/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs
@@ -5,7 +5,10 @@ using static Titano.Tools.Commands.ScreenshotsCommand;
namespace Titano.Tools.Tests;
-/// The screenshot diff fails on changed screens, tolerates noise and only warns about screens without a baseline (BRIEF §20.2).
+///
+/// The screenshot diff fails on changed screens and tolerates noise (BRIEF §20.2); a screen without a baseline fails
+/// once any baseline exists, and a run before the first baselines says that it compared nothing.
+///
public sealed class ScreenshotsCommandTests(TestContext context)
{
private const int Size = 100;
@@ -90,17 +93,47 @@ public sealed class ScreenshotsCommandTests(TestContext context)
}
[Test]
- public void AScreenWithoutABaselineOnlyWarns()
+ public void BeforeTheFirstBaselinesTheRunWarnsThatItComparedNothing()
{
var (captured, baseline, output) = Folders();
WriteScreen(captured, "welcome-dark.png");
- using var log = new StringWriter();
- var report = Run(captured, Path.Combine(baseline, "not-there"), output, log);
+ WriteScreen(captured, "about-dark.png");
+ // A README is not a baseline: the folder as committed before the first screens are accepted.
+ File.WriteAllText(Path.Combine(baseline, "README.md"), "# Screenshot baselines");
+ foreach (var folder in new[] { baseline, Path.Combine(baseline, "not-there") })
+ {
+ using var log = new StringWriter();
+ var report = Run(captured, folder, output, log);
- Assert.False(report.Fails);
- Assert.Equal(Verdict.New, report.Screens.Single().Verdict);
- Assert.Contains("::warning title=Screenshots without baseline::1 screenshot(s) have no baseline (welcome-dark.png).", log.ToString());
- Assert.Contains("0 compared", log.ToString());
+ Assert.False(report.Fails);
+ Assert.False(report.Armed);
+ Assert.True(report.Screens.All(s => s.Verdict == Verdict.New));
+ Assert.Contains("::warning title=No screenshot baselines::The baseline folder holds no PNG: nothing was compared, and no visual change can fail " +
+ "this step until baselines are committed. 2 screenshot(s) without a baseline (about-dark.png, welcome-dark.png);", log.ToString());
+ Assert.False(log.ToString().Contains("::error", StringComparison.Ordinal));
+ Assert.Contains("0 compared", log.ToString());
+ Assert.Contains("**The baseline folder holds no PNG: nothing was compared", File.ReadAllText(Path.Combine(output, "summary.md")));
+ }
+ }
+
+ [Test]
+ public void OnceAnyBaselineExistsAScreenWithoutOneFails()
+ {
+ var (captured, baseline, output) = Folders();
+ WriteScreen(baseline, "library-dark.png");
+ WriteScreen(captured, "library-dark.png");
+ // A screen added after the baselines were committed, whose baseline nobody accepted.
+ WriteScreen(captured, "library-new-dark.png");
+ using var log = new StringWriter();
+ var report = Run(captured, baseline, output, log);
+
+ Assert.True(report.Armed);
+ Assert.True(report.Fails);
+ Assert.SequenceEqual(new[] { Verdict.Same, Verdict.Unbased }, report.Screens.Select(s => s.Verdict));
+ Assert.Contains("::error title=Screenshot without baseline::library-new-dark.png: no baseline, while other screens have one (screenshots accept adds it)", log.ToString());
+ Assert.False(log.ToString().Contains("::warning", StringComparison.Ordinal), "the not-compared warning belongs to a run before the first baselines");
+ Assert.Contains("1 compared: 1 within tolerance, 0 changed; 1 without a baseline; 0 missing.", log.ToString());
+ Assert.False(File.ReadAllText(Path.Combine(output, "summary.md")).Contains("holds no PNG", StringComparison.Ordinal));
}
[Test]
@@ -138,8 +171,11 @@ public sealed class ScreenshotsCommandTests(TestContext context)
int Run(params string[] args) => ScreenshotsCommand.Run(args, _ => null, log);
Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--tolerance", "8", "--max-diff", "0.001"));
- Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05"));
+ // a.png is now within the limit, but b.png still has no baseline beside a.png's.
+ Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05"));
Assert.FileExists(Path.Combine(output, "a-diff.png"));
+ File.Copy(Path.Combine(captured, "b.png"), Path.Combine(baseline, "b.png"));
+ Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05"));
Assert.Equal(0, Run("accept", captured, "--baseline", baseline));
Assert.FileExists(Path.Combine(baseline, "b.png"));
Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output));
diff --git a/tests/visual/baselines/README.md b/tests/visual/baselines/README.md
index 8c20e44b..7ac9013f 100644
--- a/tests/visual/baselines/README.md
+++ b/tests/visual/baselines/README.md
@@ -11,7 +11,10 @@ dotnet run --project src/Titano.Tools -- screenshots diff out/screenshots --out
- A pixel differs when one of its channels moves by more than `--tolerance` (default 8 of 255); a screen fails when
more than `--max-diff` of its pixels differ (default 0.0005, that is 0.05 %), when its size changed, or when a
baseline here has no screenshot any more.
-- A screenshot without a baseline only warns.
+- A screenshot without a baseline fails as soon as this folder holds any PNG: a pull request that adds a screen adds its
+ baselines too.
+- While this folder holds no PNG nothing is compared: the step passes, with a warning and a bold line in the job summary
+ saying that no visual change can fail it.
- `out/screenshot-diff` gets `-diff.png` for every screen with differing pixels (the baseline faded to grey, the
differences magenta) and `summary.md`; CI uploads them as the `screenshot-diff` artifact and adds the summary to the
job page.
@@ -40,3 +43,10 @@ git worktree remove ../ci-screenshots
`screenshots accept ` copies every PNG of `` here. Commit only the screens whose change is intended, and delete
the baseline of a screen that no longer exists.
+
+## The first baselines
+
+None is committed yet. Screenshots captured before the capture was made reproducible cannot serve: their times and
+dates follow the moment they were taken (a job finished at "12:18 AM", sample sequences dated from that day). The first
+baselines are accepted, all at once, from the `ci-screenshots` commit of the first green `windows-latest` run whose
+code captures at `SampleData.Now`; from that commit on, every screen is gated.