using System.Globalization;
using System.Text;
using Chronos.Core.Codecs.Png;
namespace Chronos.Tools.Commands;
///
/// Visual regression for the --capture screenshots (BRIEF §20.2). screenshots diff <dir> compares every
/// PNG with the baseline of the same name (tests/visual/baselines unless --baseline): a pixel differs when
/// one of its channels moves by more than --tolerance (of 255), and a screen fails when more than --max-diff
/// of its pixels differ (a fraction), when its size changed, or when a baseline has no screenshot any more. A screen
/// without a baseline fails once the baseline folder holds any PNG, so a screen added later cannot stay ungated; before
/// the first baselines are committed nothing is compared, and the run says so in a warning and in its summary instead
/// of passing for a clean diff. Every screen with differing pixels gets <name>-diff.png in --out (the
/// baseline faded to grey, the differing pixels magenta) and the run a summary.md there, copied to the job summary
/// when the workflow runner provides one. screenshots accept <dir> copies the screenshots over the baselines.
///
internal sealed class ScreenshotsCommand : ICommand
{
/// Absorbs the last-bit noise of anti-aliased edges, not a changed colour.
public const int DefaultTolerance = 8;
///
/// 0.05 %: about 650 pixels of a 1440×900 screen. A few digits of a measured value (latency, the last comparison's
/// time) stay below it; a moved or resized control, a new colour or a missing picture do not.
///
public const double DefaultMaxDiff = 0.0005;
public string Name => "screenshots";
public string Summary => "Pixel diff of --capture screenshots against tests/visual/baselines (diff
[--baseline, --out, --tolerance, --max-diff]; accept )";
public Task RunAsync(string[] args) => Task.FromResult(Run(args, Environment.GetEnvironmentVariable, Console.Out));
/// The command with its environment passed in: tests run it without workflow commands or job summary.
internal static int Run(string[] args, Func environment, TextWriter output)
{
var a = new Args(args);
if (a.Positional is not [var verb, var directory]) throw new ToolException("Usage: screenshots diff [--baseline ] [--out ] [--tolerance 0-255] [--max-diff fraction] | screenshots accept [--baseline ]");
if (!Directory.Exists(directory)) throw new ToolException($"Screenshot folder not found: {directory}");
var baseline = a.Get("baseline") ?? Path.Combine(Args.RepoRoot(), "tests", "visual", "baselines");
switch (verb)
{
case "diff":
var options = new DiffOptions(
directory,
baseline,
a.Get("out") ?? Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + "-diff",
ParseTolerance(a.Get("tolerance")),
ParseMaxDiff(a.Get("max-diff")));
// Gitea Actions sets GITHUB_ACTIONS and understands the GitHub-compatible workflow commands.
var actions = environment("GITHUB_ACTIONS") == "true";
var summary = actions ? environment("GITHUB_STEP_SUMMARY") : null;
return Diff(options, output, annotate: actions, stepSummary: summary).Fails ? 1 : 0;
case "accept":
Directory.CreateDirectory(baseline);
var accepted = 0;
foreach (var file in Directory.EnumerateFiles(directory, "*.png").Order(StringComparer.Ordinal))
{
File.Copy(file, Path.Combine(baseline, Path.GetFileName(file)), overwrite: true);
accepted++;
}
output.WriteLine($"{accepted} screenshot(s) copied to {baseline}");
return 0;
default:
throw new ToolException($"Unknown screenshots verb '{verb}' (use diff or accept).");
}
}
internal sealed record DiffOptions(string Directory, string Baseline, string Output, int Tolerance, double MaxDiff);
internal enum Verdict
{
/// No pixel beyond the tolerance.
Same,
/// Some pixels beyond the tolerance, no more than the allowed fraction.
Within,
Changed,
Resized,
Unreadable,
/// A screenshot without a baseline while the baseline folder holds none at all: warns only.
New,
/// A screenshot without a baseline while other screens have one.
Unbased,
/// A baseline whose screen was not captured.
Missing,
}
internal sealed record ScreenResult(string Name, Verdict Verdict, long Differing, long Pixels, string? Detail = null)
{
public double Fraction => Pixels == 0 ? 0 : (double)Differing / Pixels;
public bool Fails => Verdict is Verdict.Changed or Verdict.Resized or Verdict.Unreadable or Verdict.Unbased or Verdict.Missing;
}
/// The baseline folder holds at least one PNG: from then on every screen needs its own.
internal sealed record DiffReport(IReadOnlyList Screens, bool Armed)
{
public bool Fails => Screens.Any(s => s.Fails);
}
/// RGBA, 8 bits per channel, top-down.
internal sealed record Picture(int Width, int Height, byte[] Pixels);
/// Compares the folders, writes the diff images and summary.md, and reports every screen.
internal static DiffReport Diff(DiffOptions options, TextWriter output, bool annotate, string? stepSummary)
{
var captured = PngNames(options.Directory);
var baselines = Directory.Exists(options.Baseline) ? PngNames(options.Baseline) : new SortedSet(StringComparer.OrdinalIgnoreCase);
var armed = baselines.Count > 0;
// Diffs of an earlier run must not pass for this run's.
if (Directory.Exists(options.Output))
{
foreach (var old in Directory.EnumerateFiles(options.Output, "*-diff.png")) File.Delete(old);
}
Directory.CreateDirectory(options.Output);
var results = new List();
foreach (var name in captured)
{
if (!baselines.Contains(name))
{
results.Add(new ScreenResult(name, armed ? Verdict.Unbased : Verdict.New, 0, 0));
continue;
}
results.Add(CompareFiles(name, Path.Combine(options.Baseline, name), Path.Combine(options.Directory, name), options));
}
results.AddRange(baselines.Where(b => !captured.Contains(b)).Select(b => new ScreenResult(b, Verdict.Missing, 0, 0)));
results.Sort((x, y) => StringComparer.OrdinalIgnoreCase.Compare(x.Name, y.Name));
foreach (var r in results) output.WriteLine($"{Label(r.Verdict),-11} {r.Name}{Describe(r, options)}");
var report = new DiffReport(results, armed);
var markdown = Markdown(report, options);
File.WriteAllText(Path.Combine(options.Output, "summary.md"), markdown);
if (stepSummary is not null) File.AppendAllText(stepSummary, markdown);
output.WriteLine(Totals(report));
if (annotate)
{
foreach (var r in results.Where(r => r.Fails))
{
var title = r.Verdict == Verdict.Unbased ? "Screenshot without baseline" : "Screenshot " + Label(r.Verdict);
output.WriteLine($"::error title={Escape(title, property: true)}::{Escape(r.Name + Describe(r, options), property: false)}");
}
// One warning for them all (a log of hundreds helps nobody): before the first baselines every screen is new.
var unbased = results.Where(r => r.Verdict == Verdict.New).Select(r => r.Name).ToList();
if (unbased.Count > 0)
{
var names = string.Join(", ", unbased.Take(5)) + (unbased.Count > 5 ? ", …" : string.Empty);
output.WriteLine($"::warning title={Escape("No screenshot baselines", property: true)}::" +
Escape($"{NotArmed} {unbased.Count} screenshot(s) without a baseline ({names}); tests/visual/baselines/README.md says how to add them.", property: false));
}
}
return report;
}
private static ScreenResult CompareFiles(string name, string baselinePath, string capturedPath, DiffOptions options)
{
Picture expected, actual;
try
{
expected = Read(baselinePath);
actual = Read(capturedPath);
}
catch (InvalidDataException ex)
{
return new ScreenResult(name, Verdict.Unreadable, 0, 0, ex.Message);
}
if (expected.Width != actual.Width || expected.Height != actual.Height)
return new ScreenResult(name, Verdict.Resized, 0, 0, $"{actual.Width}×{actual.Height}, baseline {expected.Width}×{expected.Height}");
var (differing, diff) = Compare(expected, actual, options.Tolerance);
var pixels = (long)actual.Width * actual.Height;
if (differing == 0) return new ScreenResult(name, Verdict.Same, 0, pixels);
File.WriteAllBytes(Path.Combine(options.Output, Path.GetFileNameWithoutExtension(name) + "-diff.png"),
PngEncoder.Encode(diff, actual.Width, actual.Height, PngColorType.Rgb, level: System.IO.Compression.CompressionLevel.Fastest));
var verdict = (double)differing / pixels > options.MaxDiff ? Verdict.Changed : Verdict.Within;
return new ScreenResult(name, verdict, differing, pixels);
}
///
/// Counts the pixels where a channel differs by more than and draws the diff image: the
/// baseline as faded grey, differing pixels magenta.
///
internal static (long Differing, byte[] Diff) Compare(Picture expected, Picture actual, int tolerance)
{
if (expected.Width != actual.Width || expected.Height != actual.Height) throw new ArgumentException("The pictures differ in size.");
var count = expected.Width * expected.Height;
var diff = new byte[count * 3];
long differing = 0;
var e = expected.Pixels;
var a = actual.Pixels;
for (var i = 0; i < count; i++)
{
var p = i * 4;
var delta = Math.Max(Math.Max(Math.Abs(e[p] - a[p]), Math.Abs(e[p + 1] - a[p + 1])), Math.Max(Math.Abs(e[p + 2] - a[p + 2]), Math.Abs(e[p + 3] - a[p + 3])));
var q = i * 3;
if (delta > tolerance)
{
differing++;
diff[q] = 255;
diff[q + 1] = 0;
diff[q + 2] = 255;
continue;
}
// Rec.709 luma of the baseline, over its alpha, squeezed into 112..207 so magenta stands out in light and dark themes.
var luma = (e[p] * 54 + e[p + 1] * 183 + e[p + 2] * 19) >> 8;
var grey = (byte)(112 + luma * e[p + 3] / 255 * 3 / 8);
diff[q] = diff[q + 1] = diff[q + 2] = grey;
}
return (differing, diff);
}
/// Decodes a PNG of any colour type and depth to (16-bit samples keep their high byte).
internal static Picture Read(string path)
{
var png = PngDecoder.Decode(File.ReadAllBytes(path));
var count = png.Width * png.Height;
var rgba = new byte[count * 4];
var channels = png.Channels;
for (var i = 0; i < count; i++)
{
byte Sample(int c) => png.Pixels8 is { } p8 ? p8[i * channels + c] : (byte)(png.Pixels16![i * channels + c] >> 8);
var (r, g, b, alpha) = channels switch
{
1 => (Sample(0), Sample(0), Sample(0), (byte)255),
2 => (Sample(0), Sample(0), Sample(0), Sample(1)),
3 => (Sample(0), Sample(1), Sample(2), (byte)255),
_ => (Sample(0), Sample(1), Sample(2), Sample(3)),
};
rgba[i * 4] = r;
rgba[i * 4 + 1] = g;
rgba[i * 4 + 2] = b;
rgba[i * 4 + 3] = alpha;
}
return new Picture(png.Width, png.Height, rgba);
}
private static SortedSet PngNames(string directory) =>
new(Directory.EnumerateFiles(directory, "*.png").Select(f => Path.GetFileName(f)), StringComparer.OrdinalIgnoreCase);
private static string Label(Verdict verdict) => verdict switch
{
Verdict.Same => "same",
Verdict.Within => "tolerated",
Verdict.Changed => "changed",
Verdict.Resized => "resized",
Verdict.Unreadable => "unreadable",
Verdict.New => "new",
Verdict.Unbased => "no baseline",
_ => "missing",
};
private static string Describe(ScreenResult r, DiffOptions options) => r.Verdict switch
{
Verdict.Same => string.Empty,
Verdict.Within or Verdict.Changed => $": {Percent(r.Fraction)} of the pixels differ ({r.Differing.ToString("N0", CultureInfo.InvariantCulture)}; limit {Percent(options.MaxDiff)})",
Verdict.Resized or Verdict.Unreadable => $": {r.Detail}",
Verdict.New => ": no baseline",
Verdict.Unbased => ": no baseline, while other screens have one (screenshots accept adds it)",
_ => ": a baseline without a screenshot",
};
/// What a run before the first baselines must not hide: it compared nothing, so it could not have failed.
private const string NotArmed = "The baseline folder holds no PNG: nothing was compared, and no visual change can fail this step until baselines are committed.";
private static string Percent(double fraction) => (fraction * 100).ToString("0.###", CultureInfo.InvariantCulture) + " %";
private static string Totals(DiffReport report)
{
int Count(params Verdict[] verdicts) => report.Screens.Count(s => verdicts.Contains(s.Verdict));
return $"{Count(Verdict.Same, Verdict.Within, Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} compared: {Count(Verdict.Same, Verdict.Within)} within tolerance, " +
$"{Count(Verdict.Changed, Verdict.Resized, Verdict.Unreadable)} changed; {Count(Verdict.New, Verdict.Unbased)} without a baseline; {Count(Verdict.Missing)} missing.";
}
private static string Markdown(DiffReport report, DiffOptions options)
{
var md = new StringBuilder();
md.AppendLine("### Screenshot diff").AppendLine();
md.AppendLine(CultureInfo.InvariantCulture, $"Tolerance {options.Tolerance} of 255 per channel, at most {Percent(options.MaxDiff)} of the pixels.").AppendLine();
if (!report.Armed) md.AppendLine(CultureInfo.InvariantCulture, $"**{NotArmed}**").AppendLine();
var listed = report.Screens.Where(s => s.Verdict != Verdict.Same).ToList();
if (listed.Count > 0)
{
md.AppendLine("| Screen | Result | Detail |").AppendLine("|---|---|---|");
foreach (var r in listed) md.AppendLine(CultureInfo.InvariantCulture, $"| {r.Name} | {Label(r.Verdict)} | {Describe(r, options).TrimStart(':', ' ')} |");
md.AppendLine();
}
md.AppendLine(Totals(report)).AppendLine();
return md.ToString();
}
private static int ParseTolerance(string? value) =>
value is null ? DefaultTolerance
: int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var t) && t is >= 0 and <= 255 ? t
: throw new ToolException($"--tolerance takes 0 to 255, not '{value}'.");
private static double ParseMaxDiff(string? value) =>
value is null ? DefaultMaxDiff
: double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var f) && f is >= 0 and <= 1 ? f
: throw new ToolException($"--max-diff takes a fraction from 0 to 1, not '{value}'.");
/// Workflow-command escaping (properties also escape ':' and ',').
private static string Escape(string value, bool property)
{
var escaped = value.Replace("%", "%25", StringComparison.Ordinal).Replace("\r", "%0D", StringComparison.Ordinal).Replace("\n", "%0A", StringComparison.Ordinal);
return property ? escaped.Replace(":", "%3A", StringComparison.Ordinal).Replace(",", "%2C", StringComparison.Ordinal) : escaped;
}
}