Solution layout, build rules with zero-package guard, in-house test framework, localization runtime, brand mark rasteriser with ICO/PNG generation, Core contracts (imaging, video, domain model, TIFF reader), Titano.Tools (make-icon, i18n, xaml-lint, package-guard) and docs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
320 lines
13 KiB
C#
320 lines
13 KiB
C#
using System.Diagnostics;
|
|
using System.Globalization;
|
|
using System.Reflection;
|
|
using System.Runtime.ExceptionServices;
|
|
using System.Text;
|
|
using System.Xml;
|
|
|
|
namespace Titano.Testing;
|
|
|
|
/// <summary>
|
|
/// Discovers and runs tests in an assembly. Test projects are console apps whose entry point is
|
|
/// <c>return await TestRunner.RunAsync(args, typeof(Program).Assembly);</c>.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// Arguments: <c>--filter text</c> (substring of the full name, or <c>category:name</c>; repeatable, OR-combined),
|
|
/// <c>--exclude text</c>, <c>--junit path</c>, <c>--parallel n</c>, <c>--list</c>, <c>--verbose</c>.
|
|
/// Exit codes: 0 all passed, 1 failures, 2 usage error or no tests matched.
|
|
/// </remarks>
|
|
public static class TestRunner
|
|
{
|
|
public static async Task<int> RunAsync(string[] args, Assembly assembly)
|
|
{
|
|
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
|
|
CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture;
|
|
|
|
Options options;
|
|
try
|
|
{
|
|
options = Options.Parse(args);
|
|
}
|
|
catch (ArgumentException ex)
|
|
{
|
|
Console.Error.WriteLine(ex.Message);
|
|
return 2;
|
|
}
|
|
|
|
var all = Discover(assembly);
|
|
var selected = all.Where(options.Matches).ToList();
|
|
if (options.List)
|
|
{
|
|
foreach (var t in selected) Console.WriteLine(t.FullName);
|
|
return 0;
|
|
}
|
|
if (selected.Count == 0)
|
|
{
|
|
Console.Error.WriteLine("No tests matched.");
|
|
return 2;
|
|
}
|
|
|
|
var stopwatch = Stopwatch.StartNew();
|
|
var results = new List<TestResult>(selected.Count);
|
|
var resultsGate = new Lock();
|
|
using var console = new ConsoleReporter(options.Verbose);
|
|
|
|
void Record(TestResult r)
|
|
{
|
|
lock (resultsGate) results.Add(r);
|
|
console.Report(r);
|
|
}
|
|
|
|
var parallel = selected.Where(t => !t.Serial).GroupBy(t => t.Type).ToList();
|
|
var serial = selected.Where(t => t.Serial).ToList();
|
|
|
|
// Classes run in parallel; tests inside a class run in declaration order.
|
|
await Parallel.ForEachAsync(
|
|
parallel,
|
|
new ParallelOptions { MaxDegreeOfParallelism = options.Parallelism },
|
|
async (group, _) =>
|
|
{
|
|
foreach (var test in group) Record(await ExecuteAsync(test).ConfigureAwait(false));
|
|
}).ConfigureAwait(false);
|
|
|
|
foreach (var test in serial) Record(await ExecuteAsync(test).ConfigureAwait(false));
|
|
|
|
stopwatch.Stop();
|
|
var passed = results.Count(r => r.Outcome == TestOutcome.Passed);
|
|
var failed = results.Count(r => r.Outcome == TestOutcome.Failed);
|
|
var skipped = results.Count(r => r.Outcome == TestOutcome.Skipped);
|
|
console.Summary(assembly.GetName().Name ?? "tests", passed, failed, skipped, stopwatch.Elapsed, results);
|
|
|
|
if (options.JUnitPath is { } junit)
|
|
WriteJUnit(junit, assembly.GetName().Name ?? "tests", results, stopwatch.Elapsed);
|
|
|
|
return failed > 0 ? 1 : 0;
|
|
}
|
|
|
|
public static IReadOnlyList<TestCase> Discover(Assembly assembly)
|
|
{
|
|
var list = new List<TestCase>();
|
|
Type[] types;
|
|
try
|
|
{
|
|
types = assembly.GetTypes();
|
|
}
|
|
catch (ReflectionTypeLoadException ex)
|
|
{
|
|
types = ex.Types.Where(t => t is not null).ToArray()!;
|
|
}
|
|
|
|
foreach (var type in types.Where(t => t is { IsClass: true, IsPublic: true } && !t.IsAbstract).OrderBy(t => t.FullName, StringComparer.Ordinal))
|
|
{
|
|
foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static).OrderBy(m => m.MetadataToken))
|
|
{
|
|
if (method.GetCustomAttribute<TestAttribute>() is not null)
|
|
{
|
|
list.Add(new TestCase(type, method, null, null));
|
|
}
|
|
else if (method.GetCustomAttribute<TheoryAttribute>() is not null)
|
|
{
|
|
foreach (var c in method.GetCustomAttributes<CaseAttribute>())
|
|
list.Add(new TestCase(type, method, c.Arguments, c.Name));
|
|
}
|
|
}
|
|
}
|
|
return list;
|
|
}
|
|
|
|
private static async Task<TestResult> ExecuteAsync(TestCase test)
|
|
{
|
|
if (test.StaticSkipReason is { } reason)
|
|
return new TestResult(test, TestOutcome.Skipped, TimeSpan.Zero, reason, null, string.Empty);
|
|
|
|
using var cts = new CancellationTokenSource();
|
|
using var context = new TestContext(test.FullName, cts.Token);
|
|
var sw = Stopwatch.StartNew();
|
|
try
|
|
{
|
|
var run = Task.Run(() => InvokeAsync(test, context), CancellationToken.None);
|
|
var finished = await Task.WhenAny(run, Task.Delay(test.TimeoutMs, CancellationToken.None)).ConfigureAwait(false);
|
|
if (finished != run)
|
|
{
|
|
await cts.CancelAsync().ConfigureAwait(false);
|
|
// Give cooperative code a moment to observe cancellation before reporting.
|
|
await Task.WhenAny(run, Task.Delay(2000, CancellationToken.None)).ConfigureAwait(false);
|
|
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, $"Timed out after {test.TimeoutMs} ms", null, context.LogText);
|
|
}
|
|
await run.ConfigureAwait(false);
|
|
return new TestResult(test, TestOutcome.Passed, sw.Elapsed, null, null, context.LogText);
|
|
}
|
|
catch (SkipException ex)
|
|
{
|
|
return new TestResult(test, TestOutcome.Skipped, sw.Elapsed, ex.Message, null, context.LogText);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
var message = ex is AssertionException ? ex.Message : $"{ex.GetType().FullName}: {ex.Message}";
|
|
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, message, ex.StackTrace, context.LogText);
|
|
}
|
|
}
|
|
|
|
private static async Task InvokeAsync(TestCase test, TestContext context)
|
|
{
|
|
object? instance = null;
|
|
try
|
|
{
|
|
if (!test.Method.IsStatic)
|
|
{
|
|
var ctor = test.Type.GetConstructor([typeof(TestContext)]);
|
|
instance = ctor is not null
|
|
? ctor.Invoke([context])
|
|
: Activator.CreateInstance(test.Type);
|
|
}
|
|
|
|
var parameters = test.Method.GetParameters();
|
|
var args = BuildArguments(test, parameters, context);
|
|
object? returned;
|
|
try
|
|
{
|
|
returned = test.Method.Invoke(instance, args);
|
|
}
|
|
catch (TargetInvocationException tie) when (tie.InnerException is not null)
|
|
{
|
|
ExceptionDispatchInfo.Capture(tie.InnerException).Throw();
|
|
throw;
|
|
}
|
|
|
|
switch (returned)
|
|
{
|
|
case Task task:
|
|
await task.ConfigureAwait(false);
|
|
break;
|
|
case ValueTask valueTask:
|
|
await valueTask.ConfigureAwait(false);
|
|
break;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
switch (instance)
|
|
{
|
|
case IAsyncDisposable ad:
|
|
await ad.DisposeAsync().ConfigureAwait(false);
|
|
break;
|
|
case IDisposable d:
|
|
d.Dispose();
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
private static object?[] BuildArguments(TestCase test, ParameterInfo[] parameters, TestContext context)
|
|
{
|
|
var supplied = test.Arguments ?? [];
|
|
var result = new object?[parameters.Length];
|
|
var next = 0;
|
|
for (var i = 0; i < parameters.Length; i++)
|
|
{
|
|
var p = parameters[i];
|
|
if (p.ParameterType == typeof(TestContext))
|
|
{
|
|
result[i] = context;
|
|
continue;
|
|
}
|
|
if (p.ParameterType == typeof(CancellationToken))
|
|
{
|
|
result[i] = context.Cancellation;
|
|
continue;
|
|
}
|
|
if (next >= supplied.Length)
|
|
throw new InvalidOperationException($"{test.FullName}: missing argument for parameter '{p.Name}'");
|
|
result[i] = Convert(supplied[next++], p.ParameterType);
|
|
}
|
|
if (next != supplied.Length)
|
|
throw new InvalidOperationException($"{test.FullName}: {supplied.Length - next} unused case argument(s)");
|
|
return result;
|
|
}
|
|
|
|
private static object? Convert(object? value, Type target)
|
|
{
|
|
if (value is null) return null;
|
|
var underlying = Nullable.GetUnderlyingType(target) ?? target;
|
|
if (underlying.IsInstanceOfType(value)) return value;
|
|
if (underlying.IsEnum) return Enum.ToObject(underlying, value);
|
|
return System.Convert.ChangeType(value, underlying, CultureInfo.InvariantCulture);
|
|
}
|
|
|
|
private static void WriteJUnit(string path, string suiteName, IReadOnlyList<TestResult> results, TimeSpan elapsed)
|
|
{
|
|
var dir = Path.GetDirectoryName(Path.GetFullPath(path));
|
|
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
|
|
var settings = new XmlWriterSettings { Indent = true, Encoding = new UTF8Encoding(false) };
|
|
using var w = XmlWriter.Create(path, settings);
|
|
w.WriteStartDocument();
|
|
w.WriteStartElement("testsuites");
|
|
w.WriteStartElement("testsuite");
|
|
w.WriteAttributeString("name", suiteName);
|
|
w.WriteAttributeString("tests", results.Count.ToString(CultureInfo.InvariantCulture));
|
|
w.WriteAttributeString("failures", results.Count(r => r.Outcome == TestOutcome.Failed).ToString(CultureInfo.InvariantCulture));
|
|
w.WriteAttributeString("skipped", results.Count(r => r.Outcome == TestOutcome.Skipped).ToString(CultureInfo.InvariantCulture));
|
|
w.WriteAttributeString("time", elapsed.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture));
|
|
foreach (var r in results.OrderBy(r => r.Test.FullName, StringComparer.Ordinal))
|
|
{
|
|
w.WriteStartElement("testcase");
|
|
w.WriteAttributeString("classname", r.Test.Type.FullName);
|
|
w.WriteAttributeString("name", r.Test.FullName[(r.Test.Type.FullName!.Length + 1)..]);
|
|
w.WriteAttributeString("time", r.Duration.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture));
|
|
if (r.Outcome == TestOutcome.Failed)
|
|
{
|
|
w.WriteStartElement("failure");
|
|
w.WriteAttributeString("message", Clean(r.Message ?? "failed"));
|
|
w.WriteString(Clean((r.StackTrace ?? string.Empty) + Environment.NewLine + r.Log));
|
|
w.WriteEndElement();
|
|
}
|
|
else if (r.Outcome == TestOutcome.Skipped)
|
|
{
|
|
w.WriteStartElement("skipped");
|
|
w.WriteAttributeString("message", Clean(r.Message ?? string.Empty));
|
|
w.WriteEndElement();
|
|
}
|
|
w.WriteEndElement();
|
|
}
|
|
w.WriteEndElement();
|
|
w.WriteEndElement();
|
|
w.WriteEndDocument();
|
|
}
|
|
|
|
private static string Clean(string s) => new(s.Where(XmlConvert.IsXmlChar).ToArray());
|
|
|
|
private sealed class Options
|
|
{
|
|
public List<string> Filters { get; } = [];
|
|
public List<string> Excludes { get; } = [];
|
|
public string? JUnitPath { get; private set; }
|
|
public int Parallelism { get; private set; } = Math.Max(1, Environment.ProcessorCount);
|
|
public bool List { get; private set; }
|
|
public bool Verbose { get; private set; }
|
|
|
|
public static Options Parse(string[] args)
|
|
{
|
|
var o = new Options();
|
|
for (var i = 0; i < args.Length; i++)
|
|
{
|
|
string Next() => i + 1 < args.Length ? args[++i] : throw new ArgumentException($"Missing value after {args[i]}");
|
|
switch (args[i])
|
|
{
|
|
case "--filter": o.Filters.Add(Next()); break;
|
|
case "--exclude": o.Excludes.Add(Next()); break;
|
|
case "--junit": o.JUnitPath = Next(); break;
|
|
case "--parallel": o.Parallelism = Math.Max(1, int.Parse(Next(), CultureInfo.InvariantCulture)); break;
|
|
case "--list": o.List = true; break;
|
|
case "--verbose": o.Verbose = true; break;
|
|
default: throw new ArgumentException($"Unknown argument {args[i]}. Use --filter, --exclude, --junit, --parallel, --list, --verbose.");
|
|
}
|
|
}
|
|
return o;
|
|
}
|
|
|
|
public bool Matches(TestCase t)
|
|
{
|
|
if (Excludes.Any(f => Match(t, f))) return false;
|
|
return Filters.Count == 0 || Filters.Any(f => Match(t, f));
|
|
}
|
|
|
|
private static bool Match(TestCase t, string filter) =>
|
|
filter.StartsWith("category:", StringComparison.OrdinalIgnoreCase)
|
|
? t.Categories.Contains(filter["category:".Length..], StringComparer.OrdinalIgnoreCase)
|
|
: t.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
}
|