using System.Diagnostics;
using System.Globalization;
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Xml;
namespace Titano.Testing;
///
/// Discovers and runs tests in an assembly. Test projects are console apps whose entry point is
/// return await TestRunner.RunAsync(args, typeof(Program).Assembly);.
///
///
/// Arguments: --filter text (substring of the full name, or category:name; repeatable, OR-combined),
/// --exclude text, --junit path, --parallel n, --list, --verbose.
/// Exit codes: 0 all passed, 1 failures, 2 usage error or no tests matched.
///
public static class TestRunner
{
public static async Task RunAsync(string[] args, Assembly assembly)
{
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture;
Options options;
try
{
options = Options.Parse(args);
}
catch (ArgumentException ex)
{
Console.Error.WriteLine(ex.Message);
return 2;
}
var all = Discover(assembly);
var selected = all.Where(options.Matches).ToList();
if (options.List)
{
foreach (var t in selected) Console.WriteLine(t.FullName);
return 0;
}
if (selected.Count == 0)
{
// An empty project is not a failure; a filter that matches nothing is a usage error.
var filtered = options.HasFilters;
Console.Error.WriteLine(filtered ? "No tests matched." : "No tests discovered.");
return filtered ? 2 : 0;
}
var stopwatch = Stopwatch.StartNew();
var results = new List(selected.Count);
var resultsGate = new Lock();
using var console = new ConsoleReporter(options.Verbose);
void Record(TestResult r)
{
lock (resultsGate) results.Add(r);
console.Report(r);
}
var parallel = selected.Where(t => !t.Serial).GroupBy(t => t.Type).ToList();
var serial = selected.Where(t => t.Serial).ToList();
// Classes run in parallel; tests inside a class run in declaration order.
await Parallel.ForEachAsync(
parallel,
new ParallelOptions { MaxDegreeOfParallelism = options.Parallelism },
async (group, _) =>
{
foreach (var test in group) Record(await ExecuteAsync(test).ConfigureAwait(false));
}).ConfigureAwait(false);
foreach (var test in serial) Record(await ExecuteAsync(test).ConfigureAwait(false));
stopwatch.Stop();
var passed = results.Count(r => r.Outcome == TestOutcome.Passed);
var failed = results.Count(r => r.Outcome == TestOutcome.Failed);
var skipped = results.Count(r => r.Outcome == TestOutcome.Skipped);
console.Summary(assembly.GetName().Name ?? "tests", passed, failed, skipped, stopwatch.Elapsed, results);
if (options.JUnitPath is { } junit)
WriteJUnit(junit, assembly.GetName().Name ?? "tests", results, stopwatch.Elapsed);
return failed > 0 ? 1 : 0;
}
public static IReadOnlyList Discover(Assembly assembly)
{
var list = new List();
Type[] types;
try
{
types = assembly.GetTypes();
}
catch (ReflectionTypeLoadException ex)
{
types = ex.Types.Where(t => t is not null).ToArray()!;
}
foreach (var type in types.Where(t => t is { IsClass: true, IsPublic: true } && !t.IsAbstract).OrderBy(t => t.FullName, StringComparer.Ordinal))
{
foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static).OrderBy(m => m.MetadataToken))
{
if (method.GetCustomAttribute() is not null)
{
list.Add(new TestCase(type, method, null, null));
}
else if (method.GetCustomAttribute() is not null)
{
foreach (var c in method.GetCustomAttributes())
list.Add(new TestCase(type, method, c.Arguments, c.Name));
}
}
}
return list;
}
private static async Task ExecuteAsync(TestCase test)
{
if (test.StaticSkipReason is { } reason)
return new TestResult(test, TestOutcome.Skipped, TimeSpan.Zero, reason, null, string.Empty);
using var cts = new CancellationTokenSource();
using var context = new TestContext(test.FullName, cts.Token);
var sw = Stopwatch.StartNew();
try
{
var run = Task.Run(() => InvokeAsync(test, context), CancellationToken.None);
var finished = await Task.WhenAny(run, Task.Delay(test.TimeoutMs, CancellationToken.None)).ConfigureAwait(false);
if (finished != run)
{
await cts.CancelAsync().ConfigureAwait(false);
// Give cooperative code a moment to observe cancellation before reporting.
await Task.WhenAny(run, Task.Delay(2000, CancellationToken.None)).ConfigureAwait(false);
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, $"Timed out after {test.TimeoutMs} ms", null, context.LogText);
}
await run.ConfigureAwait(false);
return new TestResult(test, TestOutcome.Passed, sw.Elapsed, null, null, context.LogText);
}
catch (SkipException ex)
{
return new TestResult(test, TestOutcome.Skipped, sw.Elapsed, ex.Message, null, context.LogText);
}
catch (Exception ex)
{
var message = ex is AssertionException ? ex.Message : $"{ex.GetType().FullName}: {ex.Message}";
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, message, ex.StackTrace, context.LogText);
}
}
private static async Task InvokeAsync(TestCase test, TestContext context)
{
object? instance = null;
try
{
if (!test.Method.IsStatic)
{
var ctor = test.Type.GetConstructor([typeof(TestContext)]);
instance = ctor is not null
? ctor.Invoke([context])
: Activator.CreateInstance(test.Type);
}
var parameters = test.Method.GetParameters();
var args = BuildArguments(test, parameters, context);
object? returned;
try
{
returned = test.Method.Invoke(instance, args);
}
catch (TargetInvocationException tie) when (tie.InnerException is not null)
{
ExceptionDispatchInfo.Capture(tie.InnerException).Throw();
throw;
}
switch (returned)
{
case Task task:
await task.ConfigureAwait(false);
break;
case ValueTask valueTask:
await valueTask.ConfigureAwait(false);
break;
}
}
finally
{
switch (instance)
{
case IAsyncDisposable ad:
await ad.DisposeAsync().ConfigureAwait(false);
break;
case IDisposable d:
d.Dispose();
break;
}
}
}
private static object?[] BuildArguments(TestCase test, ParameterInfo[] parameters, TestContext context)
{
var supplied = test.Arguments ?? [];
var result = new object?[parameters.Length];
var next = 0;
for (var i = 0; i < parameters.Length; i++)
{
var p = parameters[i];
if (p.ParameterType == typeof(TestContext))
{
result[i] = context;
continue;
}
if (p.ParameterType == typeof(CancellationToken))
{
result[i] = context.Cancellation;
continue;
}
if (next >= supplied.Length)
throw new InvalidOperationException($"{test.FullName}: missing argument for parameter '{p.Name}'");
result[i] = Convert(supplied[next++], p.ParameterType);
}
if (next != supplied.Length)
throw new InvalidOperationException($"{test.FullName}: {supplied.Length - next} unused case argument(s)");
return result;
}
private static object? Convert(object? value, Type target)
{
if (value is null) return null;
var underlying = Nullable.GetUnderlyingType(target) ?? target;
if (underlying.IsInstanceOfType(value)) return value;
if (underlying.IsEnum) return Enum.ToObject(underlying, value);
return System.Convert.ChangeType(value, underlying, CultureInfo.InvariantCulture);
}
private static void WriteJUnit(string path, string suiteName, IReadOnlyList results, TimeSpan elapsed)
{
var dir = Path.GetDirectoryName(Path.GetFullPath(path));
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
var settings = new XmlWriterSettings { Indent = true, Encoding = new UTF8Encoding(false) };
using var w = XmlWriter.Create(path, settings);
w.WriteStartDocument();
w.WriteStartElement("testsuites");
w.WriteStartElement("testsuite");
w.WriteAttributeString("name", suiteName);
w.WriteAttributeString("tests", results.Count.ToString(CultureInfo.InvariantCulture));
w.WriteAttributeString("failures", results.Count(r => r.Outcome == TestOutcome.Failed).ToString(CultureInfo.InvariantCulture));
w.WriteAttributeString("skipped", results.Count(r => r.Outcome == TestOutcome.Skipped).ToString(CultureInfo.InvariantCulture));
w.WriteAttributeString("time", elapsed.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture));
foreach (var r in results.OrderBy(r => r.Test.FullName, StringComparer.Ordinal))
{
w.WriteStartElement("testcase");
w.WriteAttributeString("classname", Clean(r.Test.Type.FullName!));
w.WriteAttributeString("name", Clean(r.Test.FullName[(r.Test.Type.FullName!.Length + 1)..]));
w.WriteAttributeString("time", r.Duration.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture));
if (r.Outcome == TestOutcome.Failed)
{
w.WriteStartElement("failure");
w.WriteAttributeString("message", Clean(r.Message ?? "failed"));
w.WriteString(Clean((r.StackTrace ?? string.Empty) + Environment.NewLine + r.Log));
w.WriteEndElement();
}
else if (r.Outcome == TestOutcome.Skipped)
{
w.WriteStartElement("skipped");
w.WriteAttributeString("message", Clean(r.Message ?? string.Empty));
w.WriteEndElement();
}
w.WriteEndElement();
}
w.WriteEndElement();
w.WriteEndElement();
w.WriteEndDocument();
}
/// Escapes characters XML cannot carry (theory cases may contain control characters) as \uXXXX.
private static string Clean(string s)
{
StringBuilder? sb = null;
for (var i = 0; i < s.Length; i++)
{
var c = s[i];
if (XmlConvert.IsXmlChar(c))
{
sb?.Append(c);
continue;
}
if (char.IsHighSurrogate(c) && i + 1 < s.Length && XmlConvert.IsXmlSurrogatePair(s[i + 1], c))
{
sb?.Append(c).Append(s[i + 1]);
i++;
continue;
}
sb ??= new StringBuilder(s, 0, i, s.Length + 8);
sb.Append(CultureInfo.InvariantCulture, $"\\u{(int)c:X4}");
}
return sb?.ToString() ?? s;
}
private sealed class Options
{
public List Filters { get; } = [];
public List Excludes { get; } = [];
public string? JUnitPath { get; private set; }
public int Parallelism { get; private set; } = Math.Max(1, Environment.ProcessorCount);
public bool List { get; private set; }
public bool Verbose { get; private set; }
public bool HasFilters => Filters.Count > 0 || Excludes.Count > 0;
public static Options Parse(string[] args)
{
var o = new Options();
for (var i = 0; i < args.Length; i++)
{
string Next() => i + 1 < args.Length ? args[++i] : throw new ArgumentException($"Missing value after {args[i]}");
switch (args[i])
{
case "--filter": o.Filters.Add(Next()); break;
case "--exclude": o.Excludes.Add(Next()); break;
case "--junit": o.JUnitPath = Next(); break;
case "--parallel": o.Parallelism = Math.Max(1, int.Parse(Next(), CultureInfo.InvariantCulture)); break;
case "--list": o.List = true; break;
case "--verbose": o.Verbose = true; break;
default: throw new ArgumentException($"Unknown argument {args[i]}. Use --filter, --exclude, --junit, --parallel, --list, --verbose.");
}
}
return o;
}
public bool Matches(TestCase t)
{
if (Excludes.Any(f => Match(t, f))) return false;
return Filters.Count == 0 || Filters.Any(f => Match(t, f));
}
private static bool Match(TestCase t, string filter) =>
filter.StartsWith("category:", StringComparison.OrdinalIgnoreCase)
? t.Categories.Contains(filter["category:".Length..], StringComparer.OrdinalIgnoreCase)
: t.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase);
}
}