Owner review 2, phase 2 review findings (scheduler core). - Log.Write caught only IOException. Microsoft.Extensions.Logging wraps provider failures in an AggregateException, so a log line that could not be written (in tests: the process-wide sink of a stopped server whose log folder was gone) escaped JobEngine.RunAsync's catch before the failure was stored: the job stayed running and the worker faulted. This was the cause of the intermittent JobApiTests/ArchiveModuleTests failures. Log.Write now drops any non-fatal sink failure, TeeLogSink keeps a line from the other sinks, FileLogSink drops a line it cannot write and reopens the file at the next one, and LoggerSink drops what the providers throw. The scheduler's loop relies on the same guarantee. - The daily backup loop (RunDailyAsync, IsDue, Interval, CheckInterval) had no caller since the index-backup schedule replaced it; it is removed and the DatabaseBackups summary describes the schedule. - The first copy after the upgrade no longer waits a day (or for 03:00): SystemSchedule.LastDone seeds a new row due at once when its work was last done a period or more before, and both hosts pass the newest copy in the backups folder for index-backup. - Back up now (Settings and Archive), the copy kept before a restore and a backup job that keeps fewer prune to the largest Copies kept of the backup schedules (DatabaseBackups.KeptAsync), not a fixed three. The four tooltips that promised three daily copies are reworded (en, it) and their stale translations deleted. - New tests for the scheduler loop itself: a tick, Wake, a pass that threw, and dispose during a pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit c5ad0c4c3f636129afe5dd1ce039185516da5621)
134 lines
5.9 KiB
C#
134 lines
5.9 KiB
C#
using System.Threading.Channels;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Scheduling;
|
|
using Chronos.Data.Tests.Scheduling;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Jobs;
|
|
|
|
/// <summary>
|
|
/// The job engine and the scheduler log from their error handling: a log line that cannot be written must change nothing
|
|
/// they do. It once left a failed job running (its log line threw before the state was stored, the cause of the
|
|
/// intermittent JobApiTests failures: the process-wide sink was a stopped test server's, whose log folder was gone) and
|
|
/// killed the worker. Serial: these tests replace the process-wide sink.
|
|
/// </summary>
|
|
[Serial]
|
|
[Category("jobs")]
|
|
public sealed class LoggingFailureTests(TestContext context)
|
|
{
|
|
private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero);
|
|
|
|
private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10));
|
|
|
|
/// <summary>Every line fails as Microsoft.Extensions.Logging fails when a file provider's folder is gone.</summary>
|
|
private sealed class BrokenLogging : IDisposable, ILogSink
|
|
{
|
|
private readonly ILogSink _previous = Log.Sink;
|
|
private readonly LogLevel _level = Log.MinimumLevel;
|
|
private int _lines;
|
|
|
|
public BrokenLogging()
|
|
{
|
|
Log.SetSink(this);
|
|
Log.MinimumLevel = LogLevel.Trace;
|
|
}
|
|
|
|
public int Lines => Volatile.Read(ref _lines);
|
|
|
|
public void Write(LogLevel level, string category, string message, Exception? exception)
|
|
{
|
|
Interlocked.Increment(ref _lines);
|
|
throw new AggregateException("An error occurred while writing to logger(s).", new DirectoryNotFoundException("Could not find a part of the path"));
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
Log.SetSink(_previous);
|
|
Log.MinimumLevel = _level;
|
|
}
|
|
}
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
|
|
private async Task<ChronosDatabase> OpenAsync() =>
|
|
await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
|
|
private static async Task<Job> WaitForStateAsync(ChronosDatabase db, Guid id, JobState state)
|
|
{
|
|
var until = DateTime.UtcNow + Budget;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
if (await db.Jobs.GetAsync(id) is { } job && job.State == state) return job;
|
|
await Task.Delay(20);
|
|
}
|
|
Assert.Fail($"Job {id} is {(await db.Jobs.GetAsync(id))?.State}, expected {state}");
|
|
return null!;
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailedJobIsMarkedFailedAndTheWorkerGoesOnWhenLoggingFails()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CallbackHandler(JobKind.Verify, c => c.Job.TargetLabel == "broken" ? throw new InvalidDataException("unreadable") : Task.CompletedTask);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
await engine.StartAsync();
|
|
using var logging = new BrokenLogging();
|
|
|
|
// The engine logs the failure before it stores it: that line failing once left the job running.
|
|
var broken = await engine.EnqueueAsync(JobKind.Verify, [], "broken");
|
|
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed);
|
|
Assert.Equal("unreadable", failed.Error);
|
|
Assert.Greater(logging.Lines, 0);
|
|
|
|
// The one worker is still claiming jobs.
|
|
var next = await engine.EnqueueAsync(JobKind.Verify, [], "fine");
|
|
await WaitForStateAsync(db, next.Id, JobState.Done);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheSchedulerFiresAndRecordsRunsWhenLoggingFails()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
// Never started: the queued backups stay queued.
|
|
await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "t" });
|
|
var clock = new ScheduleClock(T0);
|
|
// Disposed at the end: a loop that faulted would throw there.
|
|
await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) });
|
|
await scheduler.StartAsync(new TestScheduleHost(null), context.Cancellation);
|
|
var runs = Channel.CreateUnbounded<ScheduleDto>();
|
|
scheduler.ScheduleChanged += dto => runs.Writer.TryWrite(dto);
|
|
var schedule = Assert.NotNull((await scheduler.CreateAsync(new CreateScheduleRequest("Backup", ScheduleActions.Backup,
|
|
new RecurrenceDto("every", Every: 10, Unit: "minutes")), null, context.Cancellation)).Schedule);
|
|
// From here every line the loop logs (the claim, the outcome, a failed pass) fails.
|
|
using var logging = new BrokenLogging();
|
|
|
|
async Task<ScheduleDto> RunAsync(Guid? previousJob)
|
|
{
|
|
clock.Advance(TimeSpan.FromMinutes(10));
|
|
scheduler.Wake();
|
|
while (true)
|
|
{
|
|
var dto = await runs.Reader.ReadAsync(context.Cancellation).AsTask().WaitAsync(Budget);
|
|
if (dto.Id == schedule.Id && dto.LastOutcome == ScheduleOutcome.Queued && dto.LastJobId != previousJob) return dto;
|
|
}
|
|
}
|
|
|
|
var first = await RunAsync(null);
|
|
Assert.Equal(clock.Now, first.LastRun);
|
|
await db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: context.Cancellation);
|
|
// The loop survived the lines it could not write: the next run fires too.
|
|
var second = await RunAsync(first.LastJobId);
|
|
Assert.Equal(clock.Now, second.LastRun);
|
|
Assert.Greater(logging.Lines, 0);
|
|
}
|
|
}
|