A screenshot without a baseline only warned, so with no baseline committed every screen was "new" and the CI step could not fail, and a screen added after the first baselines could stay ungated for good. Now the baseline folder is armed as soon as it holds a PNG: from then on a screen without its own baseline fails (Verdict.Unbased, an error annotation). Before the first baselines the run says plainly, in its warning and in bold in the job summary, that it compared nothing and that no visual change can fail it. The baselines README explains why the existing ci-screenshots pictures cannot serve (captured before the clock was fixed: times and dates follow the capture moment) and how the first set is accepted from the first green windows-latest run of the reproducible capture. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
189 lines
10 KiB
C#
189 lines
10 KiB
C#
using Titano.Core.Codecs.Png;
|
||
using Titano.Testing;
|
||
using Titano.Tools.Commands;
|
||
using static Titano.Tools.Commands.ScreenshotsCommand;
|
||
|
||
namespace Titano.Tools.Tests;
|
||
|
||
/// <summary>
|
||
/// The screenshot diff fails on changed screens and tolerates noise (BRIEF §20.2); a screen without a baseline fails
|
||
/// once any baseline exists, and a run before the first baselines says that it compared nothing.
|
||
/// </summary>
|
||
public sealed class ScreenshotsCommandTests(TestContext context)
|
||
{
|
||
private const int Size = 100;
|
||
|
||
/// <summary>A 100×100 RGB screen: a dark background with a light panel, then <paramref name="change"/> applied.</summary>
|
||
private static void WriteScreen(string folder, string name, Action<byte[]>? change = null, int width = Size, int height = Size)
|
||
{
|
||
var pixels = new byte[width * height * 3];
|
||
for (var y = 0; y < height; y++)
|
||
{
|
||
for (var x = 0; x < width; x++)
|
||
{
|
||
var v = (byte)(x is >= 20 and < 80 && y is >= 30 and < 60 ? 230 : 32);
|
||
pixels[(y * width + x) * 3] = pixels[(y * width + x) * 3 + 1] = pixels[(y * width + x) * 3 + 2] = v;
|
||
}
|
||
}
|
||
change?.Invoke(pixels);
|
||
File.WriteAllBytes(Path.Combine(folder, name), PngEncoder.Encode(pixels, width, height, PngColorType.Rgb));
|
||
}
|
||
|
||
/// <summary>Adds <paramref name="delta"/> to the red channel of the first <paramref name="count"/> pixels.</summary>
|
||
private static Action<byte[]> Shift(int count, int delta) => pixels =>
|
||
{
|
||
for (var i = 0; i < count; i++) pixels[i * 3] = (byte)(pixels[i * 3] + delta);
|
||
};
|
||
|
||
private (string Captured, string Baseline, string Output) Folders()
|
||
{
|
||
var root = context.CreateTempDirectory();
|
||
var folders = (Path.Combine(root, "screenshots"), Path.Combine(root, "baselines"), Path.Combine(root, "diff"));
|
||
Directory.CreateDirectory(folders.Item1);
|
||
Directory.CreateDirectory(folders.Item2);
|
||
return folders;
|
||
}
|
||
|
||
private static DiffReport Run(string captured, string baseline, string output, StringWriter log, int tolerance = DefaultTolerance, double maxDiff = DefaultMaxDiff) =>
|
||
ScreenshotsCommand.Diff(new DiffOptions(captured, baseline, output, tolerance, maxDiff), log, annotate: true, stepSummary: null);
|
||
|
||
[Test]
|
||
public void ChangedScreensFailAndNoiseWithinTheToleranceDoesNot()
|
||
{
|
||
var (captured, baseline, output) = Folders();
|
||
foreach (var name in new[] { "library-dark.png", "jobs-dark.png", "about-dark.png", "server-dark.png" }) WriteScreen(baseline, name);
|
||
WriteScreen(captured, "library-dark.png");
|
||
// Every pixel a little off: anti-aliasing noise below the per-channel tolerance.
|
||
WriteScreen(captured, "jobs-dark.png", Shift(Size * Size, DefaultTolerance));
|
||
// Four pixels clearly off: 0.04 % of the screen, under the 0.05 % allowed.
|
||
WriteScreen(captured, "about-dark.png", Shift(4, 100));
|
||
// A whole row moved: 1 % of the screen.
|
||
WriteScreen(captured, "server-dark.png", Shift(Size, 100));
|
||
|
||
using var log = new StringWriter();
|
||
var report = Run(captured, baseline, output, log);
|
||
|
||
Assert.True(report.Fails);
|
||
Assert.SequenceEqual(new[] { Verdict.Within, Verdict.Same, Verdict.Same, Verdict.Changed }, report.Screens.Select(s => s.Verdict));
|
||
Assert.SequenceEqual(new[] { "about-dark.png", "jobs-dark.png", "library-dark.png", "server-dark.png" }, report.Screens.Select(s => s.Name));
|
||
Assert.Equal(Size, report.Screens[3].Differing);
|
||
Assert.FileExists(Path.Combine(output, "server-dark-diff.png"));
|
||
Assert.FileExists(Path.Combine(output, "about-dark-diff.png"));
|
||
Assert.FileMissing(Path.Combine(output, "library-dark-diff.png"));
|
||
Assert.Contains("::error title=Screenshot changed::server-dark.png: 1 %25 of the pixels differ (100; limit 0.05 %25)", log.ToString());
|
||
Assert.Contains("| server-dark.png | changed | 1 % of the pixels differ (100; limit 0.05 %) |", File.ReadAllText(Path.Combine(output, "summary.md")));
|
||
}
|
||
|
||
[Test]
|
||
public void TheDiffImageShowsTheDifferingPixelsInMagentaOverTheFadedBaseline()
|
||
{
|
||
var (captured, baseline, output) = Folders();
|
||
WriteScreen(baseline, "a.png");
|
||
WriteScreen(captured, "a.png", Shift(Size, 100));
|
||
using var log = new StringWriter();
|
||
Run(captured, baseline, output, log);
|
||
|
||
var diff = ScreenshotsCommand.Read(Path.Combine(output, "a-diff.png"));
|
||
Assert.Equal(Size, diff.Width);
|
||
Assert.SequenceEqual(new byte[] { 255, 0, 255, 255 }, diff.Pixels[..4], "a differing pixel");
|
||
var panel = ((40 * Size) + 50) * 4;
|
||
var background = ((90 * Size) + 50) * 4;
|
||
Assert.Greater(diff.Pixels[panel], diff.Pixels[background], "the baseline's light panel stays lighter than its background");
|
||
Assert.InRange(diff.Pixels[background], 112, 207);
|
||
}
|
||
|
||
[Test]
|
||
public void BeforeTheFirstBaselinesTheRunWarnsThatItComparedNothing()
|
||
{
|
||
var (captured, baseline, output) = Folders();
|
||
WriteScreen(captured, "welcome-dark.png");
|
||
WriteScreen(captured, "about-dark.png");
|
||
// A README is not a baseline: the folder as committed before the first screens are accepted.
|
||
File.WriteAllText(Path.Combine(baseline, "README.md"), "# Screenshot baselines");
|
||
foreach (var folder in new[] { baseline, Path.Combine(baseline, "not-there") })
|
||
{
|
||
using var log = new StringWriter();
|
||
var report = Run(captured, folder, output, log);
|
||
|
||
Assert.False(report.Fails);
|
||
Assert.False(report.Armed);
|
||
Assert.True(report.Screens.All(s => s.Verdict == Verdict.New));
|
||
Assert.Contains("::warning title=No screenshot baselines::The baseline folder holds no PNG: nothing was compared, and no visual change can fail " +
|
||
"this step until baselines are committed. 2 screenshot(s) without a baseline (about-dark.png, welcome-dark.png);", log.ToString());
|
||
Assert.False(log.ToString().Contains("::error", StringComparison.Ordinal));
|
||
Assert.Contains("0 compared", log.ToString());
|
||
Assert.Contains("**The baseline folder holds no PNG: nothing was compared", File.ReadAllText(Path.Combine(output, "summary.md")));
|
||
}
|
||
}
|
||
|
||
[Test]
|
||
public void OnceAnyBaselineExistsAScreenWithoutOneFails()
|
||
{
|
||
var (captured, baseline, output) = Folders();
|
||
WriteScreen(baseline, "library-dark.png");
|
||
WriteScreen(captured, "library-dark.png");
|
||
// A screen added after the baselines were committed, whose baseline nobody accepted.
|
||
WriteScreen(captured, "library-new-dark.png");
|
||
using var log = new StringWriter();
|
||
var report = Run(captured, baseline, output, log);
|
||
|
||
Assert.True(report.Armed);
|
||
Assert.True(report.Fails);
|
||
Assert.SequenceEqual(new[] { Verdict.Same, Verdict.Unbased }, report.Screens.Select(s => s.Verdict));
|
||
Assert.Contains("::error title=Screenshot without baseline::library-new-dark.png: no baseline, while other screens have one (screenshots accept adds it)", log.ToString());
|
||
Assert.False(log.ToString().Contains("::warning", StringComparison.Ordinal), "the not-compared warning belongs to a run before the first baselines");
|
||
Assert.Contains("1 compared: 1 within tolerance, 0 changed; 1 without a baseline; 0 missing.", log.ToString());
|
||
Assert.False(File.ReadAllText(Path.Combine(output, "summary.md")).Contains("holds no PNG", StringComparison.Ordinal));
|
||
}
|
||
|
||
[Test]
|
||
public void AResizedScreenAndABaselineWithoutItsScreenFail()
|
||
{
|
||
var (captured, baseline, output) = Folders();
|
||
WriteScreen(baseline, "settings-dark.png");
|
||
WriteScreen(baseline, "setup-failed-dark.png");
|
||
WriteScreen(captured, "settings-dark.png", width: Size + 1);
|
||
using var log = new StringWriter();
|
||
var report = Run(captured, baseline, output, log);
|
||
|
||
Assert.SequenceEqual(new[] { Verdict.Resized, Verdict.Missing }, report.Screens.Select(s => s.Verdict));
|
||
Assert.True(report.Screens.All(s => s.Fails));
|
||
Assert.Contains("settings-dark.png: 101×100, baseline 100×100", log.ToString());
|
||
}
|
||
|
||
[Test]
|
||
public void ThePerChannelToleranceIsInclusive()
|
||
{
|
||
var expected = new Picture(2, 1, [10, 10, 10, 255, 10, 10, 10, 255]);
|
||
var actual = new Picture(2, 1, [18, 10, 10, 255, 10, 10, 10, 247]);
|
||
Assert.Equal(0L, ScreenshotsCommand.Compare(expected, actual, tolerance: 8).Differing);
|
||
Assert.Equal(2L, ScreenshotsCommand.Compare(expected, actual, tolerance: 7).Differing, "alpha counts like a colour");
|
||
}
|
||
|
||
[Test]
|
||
public void TheCommandExitsWithOneOnAChangeAndAcceptCopiesTheBaselines()
|
||
{
|
||
var (captured, baseline, output) = Folders();
|
||
WriteScreen(baseline, "a.png");
|
||
WriteScreen(captured, "a.png", Shift(Size * 2, 120));
|
||
WriteScreen(captured, "b.png");
|
||
using var log = new StringWriter();
|
||
int Run(params string[] args) => ScreenshotsCommand.Run(args, _ => null, log);
|
||
|
||
Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--tolerance", "8", "--max-diff", "0.001"));
|
||
// a.png is now within the limit, but b.png still has no baseline beside a.png's.
|
||
Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05"));
|
||
Assert.FileExists(Path.Combine(output, "a-diff.png"));
|
||
File.Copy(Path.Combine(captured, "b.png"), Path.Combine(baseline, "b.png"));
|
||
Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05"));
|
||
Assert.Equal(0, Run("accept", captured, "--baseline", baseline));
|
||
Assert.FileExists(Path.Combine(baseline, "b.png"));
|
||
Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output));
|
||
// The diff of the earlier run is removed with the change it showed.
|
||
Assert.FileMissing(Path.Combine(output, "a-diff.png"));
|
||
Assert.False(log.ToString().Contains("::", StringComparison.Ordinal), "no workflow commands outside GitHub Actions");
|
||
Assert.Throws<ToolException>(() => Run("diff", captured, "--baseline", baseline, "--tolerance", "300"));
|
||
Assert.Throws<ToolException>(() => Run("compare", captured));
|
||
}
|
||
}
|