using Chronos.Core.Codecs.Png; using Chronos.Testing; using Chronos.Tools.Commands; using static Chronos.Tools.Commands.ScreenshotsCommand; namespace Chronos.Tools.Tests; /// /// The screenshot diff fails on changed screens and tolerates noise (BRIEF §20.2); a screen without a baseline fails /// once any baseline exists, and a run before the first baselines says that it compared nothing. /// public sealed class ScreenshotsCommandTests(TestContext context) { private const int Size = 100; /// A 100×100 RGB screen: a dark background with a light panel, then applied. private static void WriteScreen(string folder, string name, Action? change = null, int width = Size, int height = Size) { var pixels = new byte[width * height * 3]; for (var y = 0; y < height; y++) { for (var x = 0; x < width; x++) { var v = (byte)(x is >= 20 and < 80 && y is >= 30 and < 60 ? 230 : 32); pixels[(y * width + x) * 3] = pixels[(y * width + x) * 3 + 1] = pixels[(y * width + x) * 3 + 2] = v; } } change?.Invoke(pixels); File.WriteAllBytes(Path.Combine(folder, name), PngEncoder.Encode(pixels, width, height, PngColorType.Rgb)); } /// Adds to the red channel of the first pixels. private static Action Shift(int count, int delta) => pixels => { for (var i = 0; i < count; i++) pixels[i * 3] = (byte)(pixels[i * 3] + delta); }; private (string Captured, string Baseline, string Output) Folders() { var root = context.CreateTempDirectory(); var folders = (Path.Combine(root, "screenshots"), Path.Combine(root, "baselines"), Path.Combine(root, "diff")); Directory.CreateDirectory(folders.Item1); Directory.CreateDirectory(folders.Item2); return folders; } private static DiffReport Run(string captured, string baseline, string output, StringWriter log, int tolerance = DefaultTolerance, double maxDiff = DefaultMaxDiff) => ScreenshotsCommand.Diff(new DiffOptions(captured, baseline, output, tolerance, maxDiff), log, annotate: true, stepSummary: null); [Test] public void ChangedScreensFailAndNoiseWithinTheToleranceDoesNot() { var (captured, baseline, output) = Folders(); foreach (var name in new[] { "library-dark.png", "jobs-dark.png", "about-dark.png", "server-dark.png" }) WriteScreen(baseline, name); WriteScreen(captured, "library-dark.png"); // Every pixel a little off: anti-aliasing noise below the per-channel tolerance. WriteScreen(captured, "jobs-dark.png", Shift(Size * Size, DefaultTolerance)); // Four pixels clearly off: 0.04 % of the screen, under the 0.05 % allowed. WriteScreen(captured, "about-dark.png", Shift(4, 100)); // A whole row moved: 1 % of the screen. WriteScreen(captured, "server-dark.png", Shift(Size, 100)); using var log = new StringWriter(); var report = Run(captured, baseline, output, log); Assert.True(report.Fails); Assert.SequenceEqual(new[] { Verdict.Within, Verdict.Same, Verdict.Same, Verdict.Changed }, report.Screens.Select(s => s.Verdict)); Assert.SequenceEqual(new[] { "about-dark.png", "jobs-dark.png", "library-dark.png", "server-dark.png" }, report.Screens.Select(s => s.Name)); Assert.Equal(Size, report.Screens[3].Differing); Assert.FileExists(Path.Combine(output, "server-dark-diff.png")); Assert.FileExists(Path.Combine(output, "about-dark-diff.png")); Assert.FileMissing(Path.Combine(output, "library-dark-diff.png")); Assert.Contains("::error title=Screenshot changed::server-dark.png: 1 %25 of the pixels differ (100; limit 0.05 %25)", log.ToString()); Assert.Contains("| server-dark.png | changed | 1 % of the pixels differ (100; limit 0.05 %) |", File.ReadAllText(Path.Combine(output, "summary.md"))); } [Test] public void TheDiffImageShowsTheDifferingPixelsInMagentaOverTheFadedBaseline() { var (captured, baseline, output) = Folders(); WriteScreen(baseline, "a.png"); WriteScreen(captured, "a.png", Shift(Size, 100)); using var log = new StringWriter(); Run(captured, baseline, output, log); var diff = ScreenshotsCommand.Read(Path.Combine(output, "a-diff.png")); Assert.Equal(Size, diff.Width); Assert.SequenceEqual(new byte[] { 255, 0, 255, 255 }, diff.Pixels[..4], "a differing pixel"); var panel = ((40 * Size) + 50) * 4; var background = ((90 * Size) + 50) * 4; Assert.Greater(diff.Pixels[panel], diff.Pixels[background], "the baseline's light panel stays lighter than its background"); Assert.InRange(diff.Pixels[background], 112, 207); } [Test] public void BeforeTheFirstBaselinesTheRunWarnsThatItComparedNothing() { var (captured, baseline, output) = Folders(); WriteScreen(captured, "welcome-dark.png"); WriteScreen(captured, "about-dark.png"); // A README is not a baseline: the folder as committed before the first screens are accepted. File.WriteAllText(Path.Combine(baseline, "README.md"), "# Screenshot baselines"); foreach (var folder in new[] { baseline, Path.Combine(baseline, "not-there") }) { using var log = new StringWriter(); var report = Run(captured, folder, output, log); Assert.False(report.Fails); Assert.False(report.Armed); Assert.True(report.Screens.All(s => s.Verdict == Verdict.New)); Assert.Contains("::warning title=No screenshot baselines::The baseline folder holds no PNG: nothing was compared, and no visual change can fail " + "this step until baselines are committed. 2 screenshot(s) without a baseline (about-dark.png, welcome-dark.png);", log.ToString()); Assert.False(log.ToString().Contains("::error", StringComparison.Ordinal)); Assert.Contains("0 compared", log.ToString()); Assert.Contains("**The baseline folder holds no PNG: nothing was compared", File.ReadAllText(Path.Combine(output, "summary.md"))); } } [Test] public void OnceAnyBaselineExistsAScreenWithoutOneFails() { var (captured, baseline, output) = Folders(); WriteScreen(baseline, "library-dark.png"); WriteScreen(captured, "library-dark.png"); // A screen added after the baselines were committed, whose baseline nobody accepted. WriteScreen(captured, "library-new-dark.png"); using var log = new StringWriter(); var report = Run(captured, baseline, output, log); Assert.True(report.Armed); Assert.True(report.Fails); Assert.SequenceEqual(new[] { Verdict.Same, Verdict.Unbased }, report.Screens.Select(s => s.Verdict)); Assert.Contains("::error title=Screenshot without baseline::library-new-dark.png: no baseline, while other screens have one (screenshots accept adds it)", log.ToString()); Assert.False(log.ToString().Contains("::warning", StringComparison.Ordinal), "the not-compared warning belongs to a run before the first baselines"); Assert.Contains("1 compared: 1 within tolerance, 0 changed; 1 without a baseline; 0 missing.", log.ToString()); Assert.False(File.ReadAllText(Path.Combine(output, "summary.md")).Contains("holds no PNG", StringComparison.Ordinal)); } [Test] public void AResizedScreenAndABaselineWithoutItsScreenFail() { var (captured, baseline, output) = Folders(); WriteScreen(baseline, "settings-dark.png"); WriteScreen(baseline, "setup-failed-dark.png"); WriteScreen(captured, "settings-dark.png", width: Size + 1); using var log = new StringWriter(); var report = Run(captured, baseline, output, log); Assert.SequenceEqual(new[] { Verdict.Resized, Verdict.Missing }, report.Screens.Select(s => s.Verdict)); Assert.True(report.Screens.All(s => s.Fails)); Assert.Contains("settings-dark.png: 101×100, baseline 100×100", log.ToString()); } [Test] public void ThePerChannelToleranceIsInclusive() { var expected = new Picture(2, 1, [10, 10, 10, 255, 10, 10, 10, 255]); var actual = new Picture(2, 1, [18, 10, 10, 255, 10, 10, 10, 247]); Assert.Equal(0L, ScreenshotsCommand.Compare(expected, actual, tolerance: 8).Differing); Assert.Equal(2L, ScreenshotsCommand.Compare(expected, actual, tolerance: 7).Differing, "alpha counts like a colour"); } [Test] public void TheCommandExitsWithOneOnAChangeAndAcceptCopiesTheBaselines() { var (captured, baseline, output) = Folders(); WriteScreen(baseline, "a.png"); WriteScreen(captured, "a.png", Shift(Size * 2, 120)); WriteScreen(captured, "b.png"); using var log = new StringWriter(); int Run(params string[] args) => ScreenshotsCommand.Run(args, _ => null, log); Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--tolerance", "8", "--max-diff", "0.001")); // a.png is now within the limit, but b.png still has no baseline beside a.png's. Assert.Equal(1, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); Assert.FileExists(Path.Combine(output, "a-diff.png")); File.Copy(Path.Combine(captured, "b.png"), Path.Combine(baseline, "b.png")); Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output, "--max-diff", "0.05")); Assert.Equal(0, Run("accept", captured, "--baseline", baseline)); Assert.FileExists(Path.Combine(baseline, "b.png")); Assert.Equal(0, Run("diff", captured, "--baseline", baseline, "--out", output)); // The diff of the earlier run is removed with the change it showed. Assert.FileMissing(Path.Combine(output, "a-diff.png")); Assert.False(log.ToString().Contains("::", StringComparison.Ordinal), "no workflow commands outside a workflow run"); Assert.Throws(() => Run("diff", captured, "--baseline", baseline, "--tolerance", "300")); Assert.Throws(() => Run("compare", captured)); } }