Files
Chronos/tests/Titano.Tools.Tests/ScreenshotsCommandTests.cs
T
Claude e583b6ed74 feat(tools): screenshot diff against committed baselines, reproducible captures
BRIEF §20.2 asks for visual regression of the --capture screenshots with a
pixel-diff tolerance; CI only published them. `Titano.Tools screenshots diff
<dir>` compares every PNG with tests/visual/baselines: a pixel differs when a
channel moves by more than --tolerance (8 of 255), and a screen fails when more
than --max-diff of its pixels differ (0.05 %), when its size changed or when a
baseline lost its screen; a screen without a baseline only warns. It writes a
diff image per screen with differing pixels (the baseline faded, the differences
magenta) and a Markdown summary, also to the GitHub job summary.
`screenshots accept <dir>` copies screenshots over the baselines. The baselines
folder starts with its README; the pictures come later from ci-screenshots.

The captures were not reproducible: the sample's jobs, renders, transfers,
cards, trash and remote sequences were dated from the clock, the Library
timeline and the Archive heatmap laid out around today, the Uploader captured
in the computer's language and an indeterminate bar animated. The sample now
dates everything from SampleData.Now, --capture freezes the new AppClock there,
formats are English, and the new Motion attached property (off in the captured
windows) keeps the indeterminate sweep still in the middle.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 10:05:49 +00:00

153 lines
7.8 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Titano.Core.Codecs.Png;
using Titano.Testing;
using Titano.Tools.Commands;
using static Titano.Tools.Commands.ScreenshotsCommand;
namespace Titano.Tools.Tests;
/// <summary>The screenshot diff fails on changed screens, tolerates noise and only warns about screens without a baseline (BRIEF §20.2).</summary>
public sealed class ScreenshotsCommandTests(TestContext context)
{
private const int Size = 100;
/// <summary>A 100×100 RGB screen: a dark background with a light panel, then <paramref name="change"/> applied.</summary>
private static void WriteScreen(string folder, string name, Action<byte[]>? change = null, int width = Size, int height = Size)
{
var pixels = new byte[width * height * 3];
for (var y = 0; y < height; y++)
{
for (var x = 0; x < width; x++)
{
var v = (byte)(x is >= 20 and < 80 && y is >= 30 and < 60 ? 230 : 32);
pixels[(y * width + x) * 3] = pixels[(y * width + x) * 3 + 1] = pixels[(y * width + x) * 3 + 2] = v;
}
}
change?.Invoke(pixels);
File.WriteAllBytes(Path.Combine(folder, name), PngEncoder.Encode(pixels, width, height, PngColorType.Rgb));
}
/// <summary>Adds <paramref name="delta"/> to the red channel of the first <paramref name="count"/> pixels.</summary>
private static Action<byte[]> Shift(int count, int delta) => pixels =>
{
for (var i = 0; i < count; i++) pixels[i * 3] = (byte)(pixels[i * 3] + delta);
};
private (string Captured, string Baseline, string Output) Folders()
{
var root = context.CreateTempDirectory();
var folders = (Path.Combine(root, "screenshots"), Path.Combine(root, "baselines"), Path.Combine(root, "diff"));
Directory.CreateDirectory(folders.Item1);
Directory.CreateDirectory(folders.Item2);
return folders;
}
private static DiffReport Run(string captured, string baseline, string output, StringWriter log, int tolerance = DefaultTolerance, double maxDiff = DefaultMaxDiff) =>
ScreenshotsCommand.Diff(new DiffOptions(captured, baseline, output, tolerance, maxDiff), log, annotate: true, stepSummary: null);
[Test]
public void ChangedScreensFailAndNoiseWithinTheToleranceDoesNot()
{
var (captured, baseline, output) = Folders();
foreach (var name in new[] { "library-dark.png", "jobs-dark.png", "about-dark.png", "server-dark.png" }) WriteScreen(baseline, name);
WriteScreen(captured, "library-dark.png");
// Every pixel a little off: anti-aliasing noise below the per-channel tolerance.
WriteScreen(captured, "jobs-dark.png", Shift(Size * Size, DefaultTolerance));
// Four pixels clearly off: 0.04 % of the screen, under the 0.05 % allowed.
WriteScreen(captured, "about-dark.png", Shift(4, 100));
// A whole row moved: 1 % of the screen.
WriteScreen(captured, "server-dark.png", Shift(Size, 100));
using var log = new StringWriter();
var report = Run(captured, baseline, output, log);
Assert.True(report.Fails);
Assert.SequenceEqual(new[] { Verdict.Within, Verdict.Same, Verdict.Same, Verdict.Changed }, report.Screens.Select(s => s.Verdict));
Assert.SequenceEqual(new[] { "about-dark.png", "jobs-dark.png", "library-dark.png", "server-dark.png" }, report.Screens.Select(s => s.Name));
Assert.Equal(Size, report.Screens[3].Differing);
Assert.FileExists(Path.Combine(output, "server-dark-diff.png"));
Assert.FileExists(Path.Combine(output, "about-dark-diff.png"));
Assert.FileMissing(Path.Combine(output, "library-dark-diff.png"));
Assert.Contains("::error title=Screenshot changed::server-dark.png: 1 %25 of the pixels differ (100; limit 0.05 %25)", log.ToString());
Assert.Contains("| server-dark.png | changed | 1 % of the pixels differ (100; limit 0.05 %) |", File.ReadAllText(Path.Combine(output, "summary.md")));
}
[Test]
public void TheDiffImageShowsTheDifferingPixelsInMagentaOverTheFadedBaseline()
{
var (captured, baseline, output) = Folders();
WriteScreen(baseline, "a.png");
WriteScreen(captured, "a.png", Shift(Size, 100));
using var log = new StringWriter();
Run(captured, baseline, output, log);
var diff = ScreenshotsCommand.Read(Path.Combine(output, "a-diff.png"));
Assert.Equal(Size, diff.Width);
Assert.SequenceEqual(new byte[] { 255, 0, 255, 255 }, diff.Pixels[..4], "a differing pixel");
var panel = ((40 * Size) + 50) * 4;
var background = ((90 * Size) + 50) * 4;
Assert.Greater(diff.Pixels[panel], diff.Pixels[background], "the baseline's light panel stays lighter than its background");
Assert.InRange(diff.Pixels[background], 112, 207);
}
[Test]
public void AScreenWithoutABaselineOnlyWarns()
{
var (captured, baseline, output) = Folders();
WriteScreen(captured, "welcome-dark.png");
using var log = new StringWriter();
var report = Run(captured, Path.Combine(baseline, "not-there"), output, log);
Assert.False(report.Fails);
Assert.Equal(Verdict.New, report.Screens.Single().Verdict);
Assert.Contains("::warning title=Screenshots without baseline::1 screenshot(s) have no baseline (welcome-dark.png).", log.ToString());
Assert.Contains("0 compared", log.ToString());
}
[Test]
public void AResizedScreenAndABaselineWithoutItsScreenFail()
{
var (captured, baseline, output) = Folders();
WriteScreen(baseline, "settings-dark.png");
WriteScreen(baseline, "setup-failed-dark.png");
WriteScreen(captured, "settings-dark.png", width: Size + 1);
using var log = new StringWriter();
var report = Run(captured, baseline, output, log);
Assert.SequenceEqual(new[] { Verdict.Resized, Verdict.Missing }, report.Screens.Select(s => s.Verdict));
Assert.True(report.Screens.All(s => s.Fails));
Assert.Contains("settings-dark.png: 101×100, baseline 100×100", log.ToString());
}
[Test]
public void ThePerChannelToleranceIsInclusive()
{
var expected = new Picture(2, 1, [10, 10, 10, 255, 10, 10, 10, 255]);
var actual = new Picture(2, 1, [18, 10, 10, 255, 10, 10, 10, 247]);
Assert.Equal(0L, ScreenshotsCommand.Compare(expected, actual, tolerance: 8).Differing);
Assert.Equal(2L, ScreenshotsCommand.Compare(expected, actual, tolerance: 7).Differing, "alpha counts like a colour");
}
[Test]
public void TheCommandExitsWithOneOnAChangeAndAcceptCopiesTheBaselines()
{
var (captured, baseline, output) = Folders();
WriteScreen(baseline, "a.png");
WriteScreen(captured, "a.png", Shift(Size * 2, 120));
WriteScreen(captured, "b.png");
using var log = new StringWriter();
int Run(params string[] args) => ScreenshotsCommand.Run(args, _ => null, log);
Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--tolerance", "8", "--max-diff", "0.001"));
Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05"));
Assert.FileExists(Path.Combine(output, "a-diff.png"));
Assert.Equal(0, Run("accept", captured, "--baseline", baseline));
Assert.FileExists(Path.Combine(baseline, "b.png"));
Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output));
// The diff of the earlier run is removed with the change it showed.
Assert.FileMissing(Path.Combine(output, "a-diff.png"));
Assert.False(log.ToString().Contains("::", StringComparison.Ordinal), "no workflow commands outside GitHub Actions");
Assert.Throws<ToolException>(() => Run("diff", captured, "--baseline", baseline, "--tolerance", "300"));
Assert.Throws<ToolException>(() => Run("compare", captured));
}
}