using System.Threading.Channels; using Chronos.Core.Diagnostics; using Chronos.Core.Model; using Chronos.Data.Archiving; using Chronos.Data.Jobs; using Chronos.Data.Scheduling; using Chronos.Data.Tests.Scheduling; using Chronos.Protocol; using Chronos.Testing; namespace Chronos.Data.Tests.Jobs; /// /// The job engine and the scheduler log from their error handling: a log line that cannot be written must change nothing /// they do. It once left a failed job running (its log line threw before the state was stored, the cause of the /// intermittent JobApiTests failures: the process-wide sink was a stopped test server's, whose log folder was gone) and /// killed the worker. Serial: these tests replace the process-wide sink. /// [Serial] [Category("jobs")] public sealed class LoggingFailureTests(TestContext context) { private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero); private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10)); /// Every line fails as Microsoft.Extensions.Logging fails when a file provider's folder is gone. private sealed class BrokenLogging : IDisposable, ILogSink { private readonly ILogSink _previous = Log.Sink; private readonly LogLevel _level = Log.MinimumLevel; private int _lines; public BrokenLogging() { Log.SetSink(this); Log.MinimumLevel = LogLevel.Trace; } public int Lines => Volatile.Read(ref _lines); public void Write(LogLevel level, string category, string message, Exception? exception) { Interlocked.Increment(ref _lines); throw new AggregateException("An error occurred while writing to logger(s).", new DirectoryNotFoundException("Could not find a part of the path")); } public void Dispose() { Log.SetSink(_previous); Log.MinimumLevel = _level; } } private sealed class CallbackHandler(JobKind kind, Func run) : IJobHandler { public JobKind Kind => kind; public Task RunAsync(JobContext context) => run(context); } private async Task OpenAsync() => await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation); private static async Task WaitForStateAsync(ChronosDatabase db, Guid id, JobState state) { var until = DateTime.UtcNow + Budget; while (DateTime.UtcNow < until) { if (await db.Jobs.GetAsync(id) is { } job && job.State == state) return job; await Task.Delay(20); } Assert.Fail($"Job {id} is {(await db.Jobs.GetAsync(id))?.State}, expected {state}"); return null!; } [Test] public async Task AFailedJobIsMarkedFailedAndTheWorkerGoesOnWhenLoggingFails() { await using var db = await OpenAsync(); var handler = new CallbackHandler(JobKind.Verify, c => c.Job.TargetLabel == "broken" ? throw new InvalidDataException("unreadable") : Task.CompletedTask); await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) }); await engine.StartAsync(); using var logging = new BrokenLogging(); // The engine logs the failure before it stores it: that line failing once left the job running. var broken = await engine.EnqueueAsync(JobKind.Verify, [], "broken"); var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed); Assert.Equal("unreadable", failed.Error); Assert.Greater(logging.Lines, 0); // The one worker is still claiming jobs. var next = await engine.EnqueueAsync(JobKind.Verify, [], "fine"); await WaitForStateAsync(db, next.Id, JobState.Done); } [Test] public async Task TheSchedulerFiresAndRecordsRunsWhenLoggingFails() { await using var db = await OpenAsync(); // Never started: the queued backups stay queued. await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "t" }); var clock = new ScheduleClock(T0); // Disposed at the end: a loop that faulted would throw there. await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) }); await scheduler.StartAsync(new TestScheduleHost(null), context.Cancellation); var runs = Channel.CreateUnbounded(); scheduler.ScheduleChanged += dto => runs.Writer.TryWrite(dto); var schedule = Assert.NotNull((await scheduler.CreateAsync(new CreateScheduleRequest("Backup", ScheduleActions.Backup, new RecurrenceDto("every", Every: 10, Unit: "minutes")), null, context.Cancellation)).Schedule); // From here every line the loop logs (the claim, the outcome, a failed pass) fails. using var logging = new BrokenLogging(); async Task RunAsync(Guid? previousJob) { clock.Advance(TimeSpan.FromMinutes(10)); scheduler.Wake(); while (true) { var dto = await runs.Reader.ReadAsync(context.Cancellation).AsTask().WaitAsync(Budget); if (dto.Id == schedule.Id && dto.LastOutcome == ScheduleOutcome.Queued && dto.LastJobId != previousJob) return dto; } } var first = await RunAsync(null); Assert.Equal(clock.Now, first.LastRun); await db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: context.Cancellation); // The loop survived the lines it could not write: the next run fires too. var second = await RunAsync(first.LastJobId); Assert.Equal(clock.Now, second.LastRun); Assert.Greater(logging.Lines, 0); } }