using System.Globalization; using System.Text; using Chronos.Core.Codecs.Png; namespace Chronos.Tools.Commands; /// /// Visual regression for the --capture screenshots (BRIEF §20.2). screenshots diff <dir> compares every /// PNG with the baseline of the same name (tests/visual/baselines unless --baseline): a pixel differs when /// one of its channels moves by more than --tolerance (of 255), and a screen fails when more than --max-diff /// of its pixels differ (a fraction), when its size changed, or when a baseline has no screenshot any more. A screen /// without a baseline fails once the baseline folder holds any PNG, so a screen added later cannot stay ungated; before /// the first baselines are committed nothing is compared, and the run says so in a warning and in its summary instead /// of passing for a clean diff. Every screen with differing pixels gets <name>-diff.png in --out (the /// baseline faded to grey, the differing pixels magenta) and the run a summary.md there, copied to the job summary /// when the workflow runner provides one. screenshots accept <dir> copies the screenshots over the baselines. /// internal sealed class ScreenshotsCommand : ICommand { /// Absorbs the last-bit noise of anti-aliased edges, not a changed colour. public const int DefaultTolerance = 8; /// /// 0.05 %: about 650 pixels of a 1440×900 screen. A few digits of a measured value (latency, the last comparison's /// time) stay below it; a moved or resized control, a new colour or a missing picture do not. /// public const double DefaultMaxDiff = 0.0005; public string Name => "screenshots"; public string Summary => "Pixel diff of --capture screenshots against tests/visual/baselines (diff [--baseline, --out, --tolerance, --max-diff]; accept )"; public Task RunAsync(string[] args) => Task.FromResult(Run(args, Environment.GetEnvironmentVariable, Console.Out)); /// The command with its environment passed in: tests run it without workflow commands or job summary. internal static int Run(string[] args, Func environment, TextWriter output) { var a = new Args(args); if (a.Positional is not [var verb, var directory]) throw new ToolException("Usage: screenshots diff [--baseline ] [--out ] [--tolerance 0-255] [--max-diff fraction] | screenshots accept [--baseline ]"); if (!Directory.Exists(directory)) throw new ToolException($"Screenshot folder not found: {directory}"); var baseline = a.Get("baseline") ?? Path.Combine(Args.RepoRoot(), "tests", "visual", "baselines"); switch (verb) { case "diff": var options = new DiffOptions( directory, baseline, a.Get("out") ?? Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + "-diff", ParseTolerance(a.Get("tolerance")), ParseMaxDiff(a.Get("max-diff"))); // Gitea Actions sets GITHUB_ACTIONS and understands the GitHub-compatible workflow commands. var actions = environment("GITHUB_ACTIONS") == "true"; var summary = actions ? environment("GITHUB_STEP_SUMMARY") : null; return Diff(options, output, annotate: actions, stepSummary: summary).Fails ? 1 : 0; case "accept": Directory.CreateDirectory(baseline); var accepted = 0; foreach (var file in Directory.EnumerateFiles(directory, "*.png").Order(StringComparer.Ordinal)) { File.Copy(file, Path.Combine(baseline, Path.GetFileName(file)), overwrite: true); accepted++; } output.WriteLine($"{accepted} screenshot(s) copied to {baseline}"); return 0; default: throw new ToolException($"Unknown screenshots verb '{verb}' (use diff or accept)."); } } internal sealed record DiffOptions(string Directory, string Baseline, string Output, int Tolerance, double MaxDiff); internal enum Verdict { /// No pixel beyond the tolerance. Same, /// Some pixels beyond the tolerance, no more than the allowed fraction. Within, Changed, Resized, Unreadable, /// A screenshot without a baseline while the baseline folder holds none at all: warns only. New, /// A screenshot without a baseline while other screens have one. Unbased, /// A baseline whose screen was not captured. Missing, } internal sealed record ScreenResult(string Name, Verdict Verdict, long Differing, long Pixels, string? Detail = null) { public double Fraction => Pixels == 0 ? 0 : (double)Differing / Pixels; public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Unbased or Verdict.Missing; } /// The baseline folder holds at least one PNG: from then on every screen needs its own. internal sealed record DiffReport(IReadOnlyList Screens, bool Armed) { public bool Fails => Screens.Any(s => s.Fails); } /// RGBA, 8 bits per channel, top-down. internal sealed record Picture(int Width, int Height, byte[] Pixels); /// Compares the folders, writes the diff images and summary.md, and reports every screen. internal static DiffReport Diff(DiffOptions options, TextWriter output, bool annotate, string? stepSummary) { var captured = PngNames(options.Directory); var baselines = Directory.Exists(options.Baseline) ? PngNames(options.Baseline) : new SortedSet(StringComparer.OrdinalIgnoreCase); var armed = baselines.Count > 0; // Diffs of an earlier run must not pass for this run's. if (Directory.Exists(options.Output)) { foreach (var old in Directory.EnumerateFiles(options.Output, "*-diff.png")) File.Delete(old); } Directory.CreateDirectory(options.Output); var results = new List(); foreach (var name in captured) { if (!baselines.Contains(name)) { results.Add(new ScreenResult(name, armed ? Verdict.Unbased : Verdict.New, 0, 0)); continue; } results.Add(CompareFiles(name, Path.Combine(options.Baseline, name), Path.Combine(options.Directory, name), options)); } results.AddRange(baselines.Where(b => !captured.Contains(b)).Select(b => new ScreenResult(b, Verdict.Missing, 0, 0))); results.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.Name, y.Name)); foreach (var r in results) output.WriteLine($"{Label(r.Verdict),-11} {r.Name}{Describe(r, options)}"); var report = new DiffReport(results, armed); var markdown = Markdown(report, options); File.WriteAllText(Path.Combine(options.Output, "summary.md"), markdown); if (stepSummary is not null) File.AppendAllText(stepSummary, markdown); output.WriteLine(Totals(report)); if (annotate) { foreach (var r in results.Where(r => r.Fails)) { var title = r.Verdict == Verdict.Unbased ? "Screenshot without baseline" : "Screenshot " + Label(r.Verdict); output.WriteLine($"::error title={Escape(title, property: true)}::{Escape(r.Name + Describe(r, options), property: false)}"); } // One warning for them all (a log of hundreds helps nobody): before the first baselines every screen is new. var unbased = results.Where(r => r.Verdict == Verdict.New).Select(r => r.Name).ToList(); if (unbased.Count > 0) { var names = string.Join(", ", unbased.Take(5)) + (unbased.Count > 5 ? ", …" : string.Empty); output.WriteLine($"::warning title={Escape("No screenshot baselines", property: true)}::" + Escape($"{NotArmed} {unbased.Count} screenshot(s) without a baseline ({names}); tests/visual/baselines/README.md says how to add them.", property: false)); } } return report; } private static ScreenResult CompareFiles(string name, string baselinePath, string capturedPath, DiffOptions options) { Picture expected, actual; try { expected = Read(baselinePath); actual = Read(capturedPath); } catch (InvalidDataException ex) { return new ScreenResult(name, Verdict.Unreadable, 0, 0, ex.Message); } if (expected.Width != actual.Width || expected.Height != actual.Height) return new ScreenResult(name, Verdict.Resized, 0, 0, $"{actual.Width}×{actual.Height}, baseline {expected.Width}×{expected.Height}"); var (differing, diff) = Compare(expected, actual, options.Tolerance); var pixels = (long)actual.Width * actual.Height; if (differing == 0) return new ScreenResult(name, Verdict.Same, 0, pixels); File.WriteAllBytes(Path.Combine(options.Output, Path.GetFileNameWithoutExtension(name) + "-diff.png"), PngEncoder.Encode(diff, actual.Width, actual.Height, PngColorType.Rgb, level: System.IO.Compression.CompressionLevel.Fastest)); var verdict = (double)differing / pixels > options.MaxDiff ? Verdict.Changed : Verdict.Within; return new ScreenResult(name, verdict, differing, pixels); } /// /// Counts the pixels where a channel differs by more than and draws the diff image: the /// baseline as faded grey, differing pixels magenta. /// internal static (long Differing, byte[] Diff) Compare(Picture expected, Picture actual, int tolerance) { if (expected.Width != actual.Width || expected.Height != actual.Height) throw new ArgumentException("The pictures differ in size."); var count = expected.Width * expected.Height; var diff = new byte[count * 3]; long differing = 0; var e = expected.Pixels; var a = actual.Pixels; for (var i = 0; i < count; i++) { var p = i * 4; var delta = Math.Max(Math.Max(Math.Abs(e[p] - a[p]), Math.Abs(e[p + 1] - a[p + 1])), Math.Max(Math.Abs(e[p + 2] - a[p + 2]), Math.Abs(e[p + 3] - a[p + 3]))); var q = i * 3; if (delta > tolerance) { differing++; diff[q] = 255; diff[q + 1] = 0; diff[q + 2] = 255; continue; } // Rec.709 luma of the baseline, over its alpha, squeezed into 112..207 so magenta stands out in light and dark themes. var luma = (e[p] * 54 + e[p + 1] * 183 + e[p + 2] * 19) >> 8; var grey = (byte)(112 + luma * e[p + 3] / 255 * 3 / 8); diff[q] = diff[q + 1] = diff[q + 2] = grey; } return (differing, diff); } /// Decodes a PNG of any colour type and depth to (16-bit samples keep their high byte). internal static Picture Read(string path) { var png = PngDecoder.Decode(File.ReadAllBytes(path)); var count = png.Width * png.Height; var rgba = new byte[count * 4]; var channels = png.Channels; for (var i = 0; i < count; i++) { byte Sample(int c) => png.Pixels8 is { } p8 ? p8[i * channels + c] : (byte)(png.Pixels16![i * channels + c] >> 8); var (r, g, b, alpha) = channels switch { 1 => (Sample(0), Sample(0), Sample(0), (byte)255), 2 => (Sample(0), Sample(0), Sample(0), Sample(1)), 3 => (Sample(0), Sample(1), Sample(2), (byte)255), _ => (Sample(0), Sample(1), Sample(2), Sample(3)), }; rgba[i * 4] = r; rgba[i * 4 + 1] = g; rgba[i * 4 + 2] = b; rgba[i * 4 + 3] = alpha; } return new Picture(png.Width, png.Height, rgba); } private static SortedSet PngNames(string directory) => new(Directory.EnumerateFiles(directory, "*.png").Select(f => Path.GetFileName(f)), StringComparer.OrdinalIgnoreCase); private static string Label(Verdict verdict) => verdict switch { Verdict.Same => "same", Verdict.Within => "tolerated", Verdict.Changed => "changed", Verdict.Resized => "resized", Verdict.Unreadable => "unreadable", Verdict.New => "new", Verdict.Unbased => "no baseline", _ => "missing", }; private static string Describe(ScreenResult r, DiffOptions options) => r.Verdict switch { Verdict.Same => string.Empty, Verdict.Within or Verdict.Changed => $": {Percent(r.Fraction)} of the pixels differ ({r.Differing.ToString("N0", CultureInfo.InvariantCulture)}; limit {Percent(options.MaxDiff)})", Verdict.Resized or Verdict.Unreadable => $": {r.Detail}", Verdict.New => ": no baseline", Verdict.Unbased => ": no baseline, while other screens have one (screenshots accept adds it)", _ => ": a baseline without a screenshot", }; /// What a run before the first baselines must not hide: it compared nothing, so it could not have failed. private const string NotArmed = "The baseline folder holds no PNG: nothing was compared, and no visual change can fail this step until baselines are committed."; private static string Percent(double fraction) => (fraction * 100).ToString("0.###", CultureInfo.InvariantCulture) + " %"; private static string Totals(DiffReport report) { int Count(params Verdict[] verdicts) => report.Screens.Count(s => verdicts.Contains(s.Verdict)); return $"{Count(Verdict.Same, Verdict.Within, Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} compared: {Count(Verdict.Same, Verdict.Within)} within tolerance, " + $"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New, Verdict.Unbased)} without a baseline; {Count(Verdict.Missing)} missing."; } private static string Markdown(DiffReport report, DiffOptions options) { var md = new StringBuilder(); md.AppendLine("### Screenshot diff").AppendLine(); md.AppendLine(CultureInfo.InvariantCulture, $"Tolerance {options.Tolerance} of 255 per channel, at most {Percent(options.MaxDiff)} of the pixels.").AppendLine(); if (!report.Armed) md.AppendLine(CultureInfo.InvariantCulture, $"**{NotArmed}**").AppendLine(); var listed = report.Screens.Where(s => s.Verdict != Verdict.Same).ToList(); if (listed.Count > 0) { md.AppendLine("| Screen | Result | Detail |").AppendLine("|---|---|---|"); foreach (var r in listed) md.AppendLine(CultureInfo.InvariantCulture, $"| {r.Name} | {Label(r.Verdict)} | {Describe(r, options).TrimStart(':', ' ')} |"); md.AppendLine(); } md.AppendLine(Totals(report)).AppendLine(); return md.ToString(); } private static int ParseTolerance(string? value) => value is null ? DefaultTolerance : int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var t) && t is >= 0 and <= 255 ? t : throw new ToolException($"--tolerance takes 0 to 255, not '{value}'."); private static double ParseMaxDiff(string? value) => value is null ? DefaultMaxDiff : double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var f) && f is >= 0 and <= 1 ? f : throw new ToolException($"--max-diff takes a fraction from 0 to 1, not '{value}'."); /// Workflow-command escaping (properties also escape ':' and ','). private static string Escape(string value, bool property) { var escaped = value.Replace("%", "%25", StringComparison.Ordinal).Replace("\r", "%0D", StringComparison.Ordinal).Replace("\n", "%0A", StringComparison.Ordinal); return property ? escaped.Replace(":", "%3A", StringComparison.Ordinal).Replace(",", "%2C", StringComparison.Ordinal) : escaped; } }