using System.Diagnostics;
using System.Globalization;
using System.Reflection;
using System.Runtime.ExceptionServices;
using System.Text;
using System.Xml;
namespace Titano.Testing;
///
/// Discovers and runs tests in an assembly. Test projects are console apps whose entry point is
/// return await TestRunner.RunAsync(args, typeof(Program).Assembly);.
///
///
/// Arguments: --filter text (substring of the full name, or category:name; repeatable, OR-combined),
/// --exclude text, --junit path, --parallel n, --list, --verbose.
/// Exit codes: 0 all passed, 1 failures, 2 usage error or no tests matched.
///
public static class TestRunner
{
public static async Task RunAsync(string[] args, Assembly assembly)
{
CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture;
Options options;
try
{
options = Options.Parse(args);
}
catch (ArgumentException ex)
{
Console.Error.WriteLine(ex.Message);
return 2;
}
var all = Discover(assembly);
var selected = all.Where(options.Matches).ToList();
if (options.List)
{
foreach (var t in selected) Console.WriteLine(t.FullName);
return 0;
}
if (selected.Count == 0)
{
// An empty project is not a failure; a filter that matches nothing is a usage error.
var filtered = options.HasFilters;
Console.Error.WriteLine(filtered ? "No tests matched." : "No tests discovered.");
return filtered ? 2 : 0;
}
var stopwatch = Stopwatch.StartNew();
var results = new List(selected.Count);
var resultsGate = new Lock();
using var console = new ConsoleReporter(options.Verbose);
void Record(TestResult r)
{
lock (resultsGate) results.Add(r);
console.Report(r);
}
var parallel = selected.Where(t => !t.Serial).GroupBy(t => t.Type).ToList();
var serial = selected.Where(t => t.Serial).ToList();
// Classes run in parallel; tests inside a class run in declaration order.
await Parallel.ForEachAsync(
parallel,
new ParallelOptions { MaxDegreeOfParallelism = options.Parallelism },
async (group, _) =>
{
foreach (var test in group) Record(await ExecuteAsync(test).ConfigureAwait(false));
}).ConfigureAwait(false);
foreach (var test in serial) Record(await ExecuteAsync(test).ConfigureAwait(false));
stopwatch.Stop();
var passed = results.Count(r => r.Outcome == TestOutcome.Passed);
var failed = results.Count(r => r.Outcome == TestOutcome.Failed);
var skipped = results.Count(r => r.Outcome == TestOutcome.Skipped);
console.Summary(assembly.GetName().Name ?? "tests", passed, failed, skipped, stopwatch.Elapsed, results);
if (options.JUnitPath is { } junit)
WriteJUnit(junit, assembly.GetName().Name ?? "tests", results, stopwatch.Elapsed);
return failed > 0 ? 1 : 0;
}
public static IReadOnlyList Discover(Assembly assembly)
{
var list = new List();
Type[] types;
try
{
types = assembly.GetTypes();
}
catch (ReflectionTypeLoadException ex)
{
types = ex.Types.Where(t => t is not null).ToArray()!;
}
foreach (var type in types.Where(t => t is { IsClass: true, IsPublic: true } && !t.IsAbstract).OrderBy(t => t.FullName, StringComparer.Ordinal))
list.AddRange(Discover(type));
return list;
}
///
/// The tests of one class, in declaration order. Assembly discovery takes top-level classes only, so a class nested
/// in a test class is a fixture that runs only when a test hands it to .
///
public static IReadOnlyList Discover(Type type)
{
var list = new List();
foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static).OrderBy(m => m.MetadataToken))
{
if (method.GetCustomAttribute() is not null)
{
list.Add(new TestCase(type, method, null, null));
}
else if (method.GetCustomAttribute() is not null)
{
foreach (var c in method.GetCustomAttributes())
list.Add(new TestCase(type, method, c.Arguments, c.Name));
}
}
return list;
}
/// Runs one test (constructor, body, disposal) under its timeout and returns its outcome.
public static async Task ExecuteAsync(TestCase test)
{
ArgumentNullException.ThrowIfNull(test);
if (test.StaticSkipReason is { } reason)
return new TestResult(test, TestOutcome.Skipped, TimeSpan.Zero, reason, null, string.Empty);
using var cts = new CancellationTokenSource();
using var context = new TestContext(test.FullName, test.Serial, cts.Token);
var sw = Stopwatch.StartNew();
try
{
var run = Task.Run(() => InvokeAsync(test, context), CancellationToken.None);
var finished = await Task.WhenAny(run, Task.Delay(test.TimeoutMs, CancellationToken.None)).ConfigureAwait(false);
if (finished != run)
{
await cts.CancelAsync().ConfigureAwait(false);
// Give cooperative code a moment to observe cancellation before reporting.
await Task.WhenAny(run, Task.Delay(2000, CancellationToken.None)).ConfigureAwait(false);
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, $"Timed out after {test.TimeoutMs} ms", null, context.LogText);
}
await run.ConfigureAwait(false);
return new TestResult(test, TestOutcome.Passed, sw.Elapsed, null, null, context.LogText);
}
catch (SkipException ex)
{
return new TestResult(test, TestOutcome.Skipped, sw.Elapsed, ex.Message, null, context.LogText);
}
catch (Exception ex)
{
var message = ex is AssertionException ? ex.Message : $"{ex.GetType().FullName}: {ex.Message}";
return new TestResult(test, TestOutcome.Failed, sw.Elapsed, message, ex.StackTrace, context.LogText);
}
}
private static async Task InvokeAsync(TestCase test, TestContext context)
{
TestContext.Current = context;
object? instance = null;
try
{
if (!test.Method.IsStatic)
{
var ctor = test.Type.GetConstructor([typeof(TestContext)]);
instance = Unwrapped(() => ctor is not null
? ctor.Invoke([context])
: Activator.CreateInstance(test.Type));
}
var parameters = test.Method.GetParameters();
var args = BuildArguments(test, parameters, context);
var returned = Unwrapped(() => test.Method.Invoke(instance, args));
switch (returned)
{
case Task task:
await task.ConfigureAwait(false);
break;
case ValueTask valueTask:
await valueTask.ConfigureAwait(false);
break;
}
}
finally
{
switch (instance)
{
case IAsyncDisposable ad:
await ad.DisposeAsync().ConfigureAwait(false);
break;
case IDisposable d:
d.Dispose();
break;
}
}
}
///
/// Makes a reflection call and rethrows what the constructor or method threw, so a skip stays a skip and a failure
/// carries its own message and stack trace instead of "Exception has been thrown by the target of an invocation".
///
private static object? Unwrapped(Func