using System.Diagnostics; using System.Globalization; using System.Reflection; using System.Runtime.ExceptionServices; using System.Text; using System.Xml; namespace Titano.Testing; /// /// Discovers and runs tests in an assembly. Test projects are console apps whose entry point is /// return await TestRunner.RunAsync(args, typeof(Program).Assembly);. /// /// /// Arguments: --filter text (substring of the full name, or category:name; repeatable, OR-combined), /// --exclude text, --junit path, --parallel n, --list, --verbose. /// Exit codes: 0 all passed, 1 failures, 2 usage error or no tests matched. /// public static class TestRunner { public static async Task RunAsync(string[] args, Assembly assembly) { CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture; CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture; Options options; try { options = Options.Parse(args); } catch (ArgumentException ex) { Console.Error.WriteLine(ex.Message); return 2; } var all = Discover(assembly); var selected = all.Where(options.Matches).ToList(); if (options.List) { foreach (var t in selected) Console.WriteLine(t.FullName); return 0; } if (selected.Count == 0) { Console.Error.WriteLine("No tests matched."); return 2; } var stopwatch = Stopwatch.StartNew(); var results = new List(selected.Count); var resultsGate = new Lock(); using var console = new ConsoleReporter(options.Verbose); void Record(TestResult r) { lock (resultsGate) results.Add(r); console.Report(r); } var parallel = selected.Where(t => !t.Serial).GroupBy(t => t.Type).ToList(); var serial = selected.Where(t => t.Serial).ToList(); // Classes run in parallel; tests inside a class run in declaration order. await Parallel.ForEachAsync( parallel, new ParallelOptions { MaxDegreeOfParallelism = options.Parallelism }, async (group, _) => { foreach (var test in group) Record(await ExecuteAsync(test).ConfigureAwait(false)); }).ConfigureAwait(false); foreach (var test in serial) Record(await ExecuteAsync(test).ConfigureAwait(false)); stopwatch.Stop(); var passed = results.Count(r => r.Outcome == TestOutcome.Passed); var failed = results.Count(r => r.Outcome == TestOutcome.Failed); var skipped = results.Count(r => r.Outcome == TestOutcome.Skipped); console.Summary(assembly.GetName().Name ?? "tests", passed, failed, skipped, stopwatch.Elapsed, results); if (options.JUnitPath is { } junit) WriteJUnit(junit, assembly.GetName().Name ?? "tests", results, stopwatch.Elapsed); return failed > 0 ? 1 : 0; } public static IReadOnlyList Discover(Assembly assembly) { var list = new List(); Type[] types; try { types = assembly.GetTypes(); } catch (ReflectionTypeLoadException ex) { types = ex.Types.Where(t => t is not null).ToArray()!; } foreach (var type in types.Where(t => t is { IsClass: true, IsPublic: true } && !t.IsAbstract).OrderBy(t => t.FullName, StringComparer.Ordinal)) { foreach (var method in type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static).OrderBy(m => m.MetadataToken)) { if (method.GetCustomAttribute() is not null) { list.Add(new TestCase(type, method, null, null)); } else if (method.GetCustomAttribute() is not null) { foreach (var c in method.GetCustomAttributes()) list.Add(new TestCase(type, method, c.Arguments, c.Name)); } } } return list; } private static async Task ExecuteAsync(TestCase test) { if (test.StaticSkipReason is { } reason) return new TestResult(test, TestOutcome.Skipped, TimeSpan.Zero, reason, null, string.Empty); using var cts = new CancellationTokenSource(); using var context = new TestContext(test.FullName, cts.Token); var sw = Stopwatch.StartNew(); try { var run = Task.Run(() => InvokeAsync(test, context), CancellationToken.None); var finished = await Task.WhenAny(run, Task.Delay(test.TimeoutMs, CancellationToken.None)).ConfigureAwait(false); if (finished != run) { await cts.CancelAsync().ConfigureAwait(false); // Give cooperative code a moment to observe cancellation before reporting. await Task.WhenAny(run, Task.Delay(2000, CancellationToken.None)).ConfigureAwait(false); return new TestResult(test, TestOutcome.Failed, sw.Elapsed, $"Timed out after {test.TimeoutMs} ms", null, context.LogText); } await run.ConfigureAwait(false); return new TestResult(test, TestOutcome.Passed, sw.Elapsed, null, null, context.LogText); } catch (SkipException ex) { return new TestResult(test, TestOutcome.Skipped, sw.Elapsed, ex.Message, null, context.LogText); } catch (Exception ex) { var message = ex is AssertionException ? ex.Message : $"{ex.GetType().FullName}: {ex.Message}"; return new TestResult(test, TestOutcome.Failed, sw.Elapsed, message, ex.StackTrace, context.LogText); } } private static async Task InvokeAsync(TestCase test, TestContext context) { object? instance = null; try { if (!test.Method.IsStatic) { var ctor = test.Type.GetConstructor([typeof(TestContext)]); instance = ctor is not null ? ctor.Invoke([context]) : Activator.CreateInstance(test.Type); } var parameters = test.Method.GetParameters(); var args = BuildArguments(test, parameters, context); object? returned; try { returned = test.Method.Invoke(instance, args); } catch (TargetInvocationException tie) when (tie.InnerException is not null) { ExceptionDispatchInfo.Capture(tie.InnerException).Throw(); throw; } switch (returned) { case Task task: await task.ConfigureAwait(false); break; case ValueTask valueTask: await valueTask.ConfigureAwait(false); break; } } finally { switch (instance) { case IAsyncDisposable ad: await ad.DisposeAsync().ConfigureAwait(false); break; case IDisposable d: d.Dispose(); break; } } } private static object?[] BuildArguments(TestCase test, ParameterInfo[] parameters, TestContext context) { var supplied = test.Arguments ?? []; var result = new object?[parameters.Length]; var next = 0; for (var i = 0; i < parameters.Length; i++) { var p = parameters[i]; if (p.ParameterType == typeof(TestContext)) { result[i] = context; continue; } if (p.ParameterType == typeof(CancellationToken)) { result[i] = context.Cancellation; continue; } if (next >= supplied.Length) throw new InvalidOperationException($"{test.FullName}: missing argument for parameter '{p.Name}'"); result[i] = Convert(supplied[next++], p.ParameterType); } if (next != supplied.Length) throw new InvalidOperationException($"{test.FullName}: {supplied.Length - next} unused case argument(s)"); return result; } private static object? Convert(object? value, Type target) { if (value is null) return null; var underlying = Nullable.GetUnderlyingType(target) ?? target; if (underlying.IsInstanceOfType(value)) return value; if (underlying.IsEnum) return Enum.ToObject(underlying, value); return System.Convert.ChangeType(value, underlying, CultureInfo.InvariantCulture); } private static void WriteJUnit(string path, string suiteName, IReadOnlyList results, TimeSpan elapsed) { var dir = Path.GetDirectoryName(Path.GetFullPath(path)); if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir); var settings = new XmlWriterSettings { Indent = true, Encoding = new UTF8Encoding(false) }; using var w = XmlWriter.Create(path, settings); w.WriteStartDocument(); w.WriteStartElement("testsuites"); w.WriteStartElement("testsuite"); w.WriteAttributeString("name", suiteName); w.WriteAttributeString("tests", results.Count.ToString(CultureInfo.InvariantCulture)); w.WriteAttributeString("failures", results.Count(r => r.Outcome == TestOutcome.Failed).ToString(CultureInfo.InvariantCulture)); w.WriteAttributeString("skipped", results.Count(r => r.Outcome == TestOutcome.Skipped).ToString(CultureInfo.InvariantCulture)); w.WriteAttributeString("time", elapsed.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture)); foreach (var r in results.OrderBy(r => r.Test.FullName, StringComparer.Ordinal)) { w.WriteStartElement("testcase"); w.WriteAttributeString("classname", r.Test.Type.FullName); w.WriteAttributeString("name", r.Test.FullName[(r.Test.Type.FullName!.Length + 1)..]); w.WriteAttributeString("time", r.Duration.TotalSeconds.ToString("F3", CultureInfo.InvariantCulture)); if (r.Outcome == TestOutcome.Failed) { w.WriteStartElement("failure"); w.WriteAttributeString("message", Clean(r.Message ?? "failed")); w.WriteString(Clean((r.StackTrace ?? string.Empty) + Environment.NewLine + r.Log)); w.WriteEndElement(); } else if (r.Outcome == TestOutcome.Skipped) { w.WriteStartElement("skipped"); w.WriteAttributeString("message", Clean(r.Message ?? string.Empty)); w.WriteEndElement(); } w.WriteEndElement(); } w.WriteEndElement(); w.WriteEndElement(); w.WriteEndDocument(); } private static string Clean(string s) => new(s.Where(XmlConvert.IsXmlChar).ToArray()); private sealed class Options { public List Filters { get; } = []; public List Excludes { get; } = []; public string? JUnitPath { get; private set; } public int Parallelism { get; private set; } = Math.Max(1, Environment.ProcessorCount); public bool List { get; private set; } public bool Verbose { get; private set; } public static Options Parse(string[] args) { var o = new Options(); for (var i = 0; i < args.Length; i++) { string Next() => i + 1 < args.Length ? args[++i] : throw new ArgumentException($"Missing value after {args[i]}"); switch (args[i]) { case "--filter": o.Filters.Add(Next()); break; case "--exclude": o.Excludes.Add(Next()); break; case "--junit": o.JUnitPath = Next(); break; case "--parallel": o.Parallelism = Math.Max(1, int.Parse(Next(), CultureInfo.InvariantCulture)); break; case "--list": o.List = true; break; case "--verbose": o.Verbose = true; break; default: throw new ArgumentException($"Unknown argument {args[i]}. Use --filter, --exclude, --junit, --parallel, --list, --verbose."); } } return o; } public bool Matches(TestCase t) { if (Excludes.Any(f => Match(t, f))) return false; return Filters.Count == 0 || Filters.Any(f => Match(t, f)); } private static bool Match(TestCase t, string filter) => filter.StartsWith("category:", StringComparison.OrdinalIgnoreCase) ? t.Categories.Contains(filter["category:".Length..], StringComparer.OrdinalIgnoreCase) : t.FullName.Contains(filter, StringComparison.OrdinalIgnoreCase); } }