Files
Chronos/tests/Titano.Testing/TestRunner.cs
T
Claude 2f978d1c80 test: run process-wide tests serially and report constructor failures as themselves
Four App tests switched the process-wide UI language inside parallel classes, so
another class's English assertion, or its constructor's reset to English, could
land in the middle of them; they now live in the serial ArchiveLanguageTests and
ImportLanguageTests. The App and Core test programs fail a test that leaves
English outside a serial class at the switch, through the new
TestContext.Current and TestContext.IsSerial.

The FrameWindow cancellation test counted unobserved task exceptions while other
classes ran beside it; it moves to the serial FrameWindowIsolationTests, and both
unobserved-exception tests drain the finalizer queue before they start counting.

The runner unwrapped TargetInvocationException only around the test method: a
skip or an assertion thrown by a test class constructor or field initializer was
reported as a failure reading "Exception has been thrown by the target of an
invocation". Constructors are unwrapped the same way now. TestRunner.Discover(Type)
and ExecuteAsync are public, so Tools.Tests checks the runner with nested fixtures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 10:05:37 +00:00

365 lines
15 KiB
C#

using System.Diagnostics;
using System.Globalization;
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Xml;
namespace Titano.Testing;
/// <summary>
/// Discovers and runs tests in an assembly. Test projects are console apps whose entry point is
/// <c>return await TestRunner.RunAsync(args, typeof(Program).Assembly);</c>.
/// </summary>
/// <remarks>
/// Arguments: <c>--filter text</c> (substring of the full name, or <c>category:name</c>; repeatable, OR-combined),
/// <c>--exclude text</c>, <c>--junit path</c>, <c>--parallel n</c>, <c>--list</c>, <c>--verbose</c>.
/// Exit codes: 0 all passed, 1 failures, 2 usage error or no tests matched.
/// </remarks>
public static class TestRunner
{
public static async Task<int> RunAsync(string[] args, Assembly assembly)
{
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture;
Options options;
try
{
options = Options.Parse(args);
}
catch (ArgumentException ex)
{
Console.Error.WriteLine(ex.Message);
return 2;
}
var all = Discover(assembly);
var selected = all.Where(options.Matches).ToList();
if (options.List)
{
foreach (var t in selected) Console.WriteLine(t.FullName);
return 0;
}
if (selected.Count == 0)
{
// An empty project is not a failure; a filter that matches nothing is a usage error.
var filtered = options.HasFilters;
Console.Error.WriteLine(filtered ? "No tests matched." : "No tests discovered.");
return filtered ? 2 : 0;
}
var stopwatch = Stopwatch.StartNew();
var results = new List<TestResult>(selected.Count);
var resultsGate = new Lock();
using var console = new ConsoleReporter(options.Verbose);
void Record(TestResult r)
{
lock (resultsGate) results.Add(r);
console.Report(r);
}
var parallel = selected.Where(t => !t.Serial).GroupBy(t => t.Type).ToList();
var serial = selected.Where(t => t.Serial).ToList();
// Classes run in parallel; tests inside a class run in declaration order.
await Parallel.ForEachAsync(
parallel,
new ParallelOptions { MaxDegreeOfParallelism = options.Parallelism },
async (group, _) =>
{
foreach (var test in group) Record(await ExecuteAsync(test).ConfigureAwait(false));
}).ConfigureAwait(false);
foreach (var test in serial) Record(await ExecuteAsync(test).ConfigureAwait(false));
stopwatch.Stop();
var passed = results.Count(r => r.Outcome == TestOutcome.Passed);
var failed = results.Count(r => r.Outcome == TestOutcome.Failed);
var skipped = results.Count(r => r.Outcome == TestOutcome.Skipped);
console.Summary(assembly.GetName().Name ?? "tests", passed, failed, skipped, stopwatch.Elapsed, results);
if (options.JUnitPath is { } junit)
WriteJUnit(junit, assembly.GetName().Name ?? "tests", results, stopwatch.Elapsed);
return failed > 0 ? 1 : 0;
}
public static IReadOnlyList<TestCase> Discover(Assembly assembly)
{
var list = new List<TestCase>();
Type[] types;
try
{
types = assembly.GetTypes();
}
catch (ReflectionTypeLoadException ex)
{
types = ex.Types.Where(t => t is not null).ToArray()!;
}
foreach (var type in types.Where(t => t is { IsClass: true, IsPublic: true } && !t.IsAbstract).OrderBy(t => t.FullName, StringComparer.Ordinal))
list.AddRange(Discover(type));
return list;
}
/// <summary>
/// The tests of one class, in declaration order. Assembly discovery takes top-level classes only, so a class nested
/// in a test class is a fixture that runs only when a test hands it to <see cref="ExecuteAsync"/>.
/// </summary>
public static IReadOnlyList<TestCase> Discover(Type type)
{
var list = new List<TestCase>();
foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static).OrderBy(m => m.MetadataToken))
{
if (method.GetCustomAttribute<TestAttribute>() is not null)
{
list.Add(new TestCase(type, method, null, null));
}
else if (method.GetCustomAttribute<TheoryAttribute>() is not null)
{
foreach (var c in method.GetCustomAttributes<CaseAttribute>())
list.Add(new TestCase(type, method, c.Arguments, c.Name));
}
}
return list;
}
/// <summary>Runs one test (constructor, body, disposal) under its timeout and returns its outcome.</summary>
public static async Task<TestResult> ExecuteAsync(TestCase test)
{
ArgumentNullException.ThrowIfNull(test);
if (test.StaticSkipReason is { } reason)
return new TestResult(test, TestOutcome.Skipped, TimeSpan.Zero, reason, null, string.Empty);
using var cts = new CancellationTokenSource();
using var context = new TestContext(test.FullName, test.Serial, cts.Token);
var sw = Stopwatch.StartNew();
try
{
var run = Task.Run(() => InvokeAsync(test, context), CancellationToken.None);
var finished = await Task.WhenAny(run, Task.Delay(test.TimeoutMs, CancellationToken.None)).ConfigureAwait(false);
if (finished != run)
{
await cts.CancelAsync().ConfigureAwait(false);
// Give cooperative code a moment to observe cancellation before reporting.
await Task.WhenAny(run, Task.Delay(2000, CancellationToken.None)).ConfigureAwait(false);
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, $"Timed out after {test.TimeoutMs} ms", null, context.LogText);
}
await run.ConfigureAwait(false);
return new TestResult(test, TestOutcome.Passed, sw.Elapsed, null, null, context.LogText);
}
catch (SkipException ex)
{
return new TestResult(test, TestOutcome.Skipped, sw.Elapsed, ex.Message, null, context.LogText);
}
catch (Exception ex)
{
var message = ex is AssertionException ? ex.Message : $"{ex.GetType().FullName}: {ex.Message}";
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, message, ex.StackTrace, context.LogText);
}
}
private static async Task InvokeAsync(TestCase test, TestContext context)
{
TestContext.Current = context;
object? instance = null;
try
{
if (!test.Method.IsStatic)
{
var ctor = test.Type.GetConstructor([typeof(TestContext)]);
instance = Unwrapped(() => ctor is not null
? ctor.Invoke([context])
: Activator.CreateInstance(test.Type));
}
var parameters = test.Method.GetParameters();
var args = BuildArguments(test, parameters, context);
var returned = Unwrapped(() => test.Method.Invoke(instance, args));
switch (returned)
{
case Task task:
await task.ConfigureAwait(false);
break;
case ValueTask valueTask:
await valueTask.ConfigureAwait(false);
break;
}
}
finally
{
switch (instance)
{
case IAsyncDisposable ad:
await ad.DisposeAsync().ConfigureAwait(false);
break;
case IDisposable d:
d.Dispose();
break;
}
}
}
/// <summary>
/// Makes a reflection call and rethrows what the constructor or method threw, so a skip stays a skip and a failure
/// carries its own message and stack trace instead of "Exception has been thrown by the target of an invocation".
/// </summary>
private static object? Unwrapped(Func<object?> call)
{
try
{
return call();
}
catch (TargetInvocationException tie) when (tie.InnerException is not null)
{
ExceptionDispatchInfo.Capture(tie.InnerException).Throw();
throw;
}
}
private static object?[] BuildArguments(TestCase test, ParameterInfo[] parameters, TestContext context)
{
var supplied = test.Arguments ?? [];
var result = new object?[parameters.Length];
var next = 0;
for (var i = 0; i < parameters.Length; i++)
{
var p = parameters[i];
if (p.ParameterType == typeof(TestContext))
{
result[i] = context;
continue;
}
if (p.ParameterType == typeof(CancellationToken))
{
result[i] = context.Cancellation;
continue;
}
if (next >= supplied.Length)
throw new InvalidOperationException($"{test.FullName}: missing argument for parameter '{p.Name}'");
result[i] = Convert(supplied[next++], p.ParameterType);
}
if (next != supplied.Length)
throw new InvalidOperationException($"{test.FullName}: {supplied.Length - next} unused case argument(s)");
return result;
}
private static object? Convert(object? value, Type target)
{
if (value is null) return null;
var underlying = Nullable.GetUnderlyingType(target) ?? target;
if (underlying.IsInstanceOfType(value)) return value;
if (underlying.IsEnum) return Enum.ToObject(underlying, value);
return System.Convert.ChangeType(value, underlying, CultureInfo.InvariantCulture);
}
private static void WriteJUnit(string path, string suiteName, IReadOnlyList<TestResult> results, TimeSpan elapsed)
{
var dir = Path.GetDirectoryName(Path.GetFullPath(path));
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
var settings = new XmlWriterSettings { Indent = true, Encoding = new UTF8Encoding(false) };
using var w = XmlWriter.Create(path, settings);
w.WriteStartDocument();
w.WriteStartElement("testsuites");
w.WriteStartElement("testsuite");
w.WriteAttributeString("name", suiteName);
w.WriteAttributeString("tests", results.Count.ToString(CultureInfo.InvariantCulture));
w.WriteAttributeString("failures", results.Count(r => r.Outcome == TestOutcome.Failed).ToString(CultureInfo.InvariantCulture));
w.WriteAttributeString("skipped", results.Count(r => r.Outcome == TestOutcome.Skipped).ToString(CultureInfo.InvariantCulture));
w.WriteAttributeString("time", elapsed.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture));
foreach (var r in results.OrderBy(r => r.Test.FullName, StringComparer.Ordinal))
{
w.WriteStartElement("testcase");
w.WriteAttributeString("classname", Clean(r.Test.Type.FullName!));
w.WriteAttributeString("name", Clean(r.Test.FullName[(r.Test.Type.FullName!.Length + 1)..]));
w.WriteAttributeString("time", r.Duration.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture));
if (r.Outcome == TestOutcome.Failed)
{
w.WriteStartElement("failure");
w.WriteAttributeString("message", Clean(r.Message ?? "failed"));
w.WriteString(Clean((r.StackTrace ?? string.Empty) + Environment.NewLine + r.Log));
w.WriteEndElement();
}
else if (r.Outcome == TestOutcome.Skipped)
{
w.WriteStartElement("skipped");
w.WriteAttributeString("message", Clean(r.Message ?? string.Empty));
w.WriteEndElement();
}
w.WriteEndElement();
}
w.WriteEndElement();
w.WriteEndElement();
w.WriteEndDocument();
}
/// <summary>Escapes characters XML cannot carry (theory cases may contain control characters) as <c>\uXXXX</c>.</summary>
private static string Clean(string s)
{
StringBuilder? sb = null;
for (var i = 0; i < s.Length; i++)
{
var c = s[i];
if (XmlConvert.IsXmlChar(c))
{
sb?.Append(c);
continue;
}
if (char.IsHighSurrogate(c) && i + 1 < s.Length && XmlConvert.IsXmlSurrogatePair(s[i + 1], c))
{
sb?.Append(c).Append(s[i + 1]);
i++;
continue;
}
sb ??= new StringBuilder(s, 0, i, s.Length + 8);
sb.Append(CultureInfo.InvariantCulture, $"\\u{(int)c:X4}");
}
return sb?.ToString() ?? s;
}
private sealed class Options
{
public List<string> Filters { get; } = [];
public List<string> Excludes { get; } = [];
public string? JUnitPath { get; private set; }
public int Parallelism { get; private set; } = Math.Max(1, Environment.ProcessorCount);
public bool List { get; private set; }
public bool Verbose { get; private set; }
public bool HasFilters => Filters.Count > 0 || Excludes.Count > 0;
public static Options Parse(string[] args)
{
var o = new Options();
for (var i = 0; i < args.Length; i++)
{
string Next() => i + 1 < args.Length ? args[++i] : throw new ArgumentException($"Missing value after {args[i]}");
switch (args[i])
{
case "--filter": o.Filters.Add(Next()); break;
case "--exclude": o.Excludes.Add(Next()); break;
case "--junit": o.JUnitPath = Next(); break;
case "--parallel": o.Parallelism = Math.Max(1, int.Parse(Next(), CultureInfo.InvariantCulture)); break;
case "--list": o.List = true; break;
case "--verbose": o.Verbose = true; break;
default: throw new ArgumentException($"Unknown argument {args[i]}. Use --filter, --exclude, --junit, --parallel, --list, --verbose.");
}
}
return o;
}
public bool Matches(TestCase t)
{
if (Excludes.Any(f => Match(t, f))) return false;
return Filters.Count == 0 || Filters.Any(f => Match(t, f));
}
private static bool Match(TestCase t, string filter) =>
filter.StartsWith("category:", StringComparison.OrdinalIgnoreCase)
? t.Categories.Contains(filter["category:".Length..], StringComparer.OrdinalIgnoreCase)
: t.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase);
}
}