fix: logging never fails a job, one backup retention, first copy after the upgrade
Owner review 2, phase 2 review findings (scheduler core). - Log.Write caught only IOException. Microsoft.Extensions.Logging wraps provider failures in an AggregateException, so a log line that could not be written (in tests: the process-wide sink of a stopped server whose log folder was gone) escaped JobEngine.RunAsync's catch before the failure was stored: the job stayed running and the worker faulted. This was the cause of the intermittent JobApiTests/ArchiveModuleTests failures. Log.Write now drops any non-fatal sink failure, TeeLogSink keeps a line from the other sinks, FileLogSink drops a line it cannot write and reopens the file at the next one, and LoggerSink drops what the providers throw. The scheduler's loop relies on the same guarantee. - The daily backup loop (RunDailyAsync, IsDue, Interval, CheckInterval) had no caller since the index-backup schedule replaced it; it is removed and the DatabaseBackups summary describes the schedule. - The first copy after the upgrade no longer waits a day (or for 03:00): SystemSchedule.LastDone seeds a new row due at once when its work was last done a period or more before, and both hosts pass the newest copy in the backups folder for index-backup. - Back up now (Settings and Archive), the copy kept before a restore and a backup job that keeps fewer prune to the largest Copies kept of the backup schedules (DatabaseBackups.KeptAsync), not a fixed three. The four tooltips that promised three daily copies are reworded (en, it) and their stale translations deleted. - New tests for the scheduler loop itself: a tick, Wake, a pass that threw, and dispose during a pass. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit c5ad0c4c3f636129afe5dd1ce039185516da5621)
This commit is contained in:
1 parent
422f33d5f3
commit
e384564784
59 files changed
+668
-264
No files matched your search
@@ -60,6 +60,12 @@ public sealed class HousekeepingJobTests(TestContext context)
|
||||
Assert.Contains(backups[0].Path, done.LogTail);
|
||||
Assert.Equal((1L, 1L), (done.ProgressDone, done.ProgressTotal));
|
||||
Assert.Null(DatabaseBackups.Unavailable(fx.Db));
|
||||
|
||||
// One folder, one retention: a backup that keeps fewer never deletes the copies the index backup schedule keeps.
|
||||
await fx.Db.Schedules.EnsureSystemAsync([new Data.Scheduling.SystemSchedule(DatabaseBackups.ScheduleKey, "Index backup", Data.Scheduling.ScheduleActions.Backup,
|
||||
Data.Scheduling.ScheduleRecurrence.Parse("daily:03:00"), new Protocol.ScheduleParametersDto(Keep: 4))], DateTimeOffset.UtcNow, fx.Ct);
|
||||
for (var i = 0; i < 3; i++) await RunAsync(fx, engine, ArchiveJobs.Backup(keep: 1));
|
||||
Assert.Count(4, DatabaseBackups.List(DatabaseBackups.DefaultDirectory(path), path));
|
||||
}
|
||||
|
||||
[Test]
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Threading.Channels;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data.Archiving;
|
||||
@@ -12,7 +13,10 @@ using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Data.Tests.Scheduling;
|
||||
|
||||
/// <summary>A clock the test sets by hand; its timers are real ones, so the scheduler's loop waits in real time (tests drive passes themselves).</summary>
|
||||
/// <summary>
|
||||
/// A clock the test sets by hand; its timers are real ones, so the scheduler's loop waits in real time (most tests drive
|
||||
/// passes themselves; the loop's own tests give it a short tick or wake it).
|
||||
/// </summary>
|
||||
internal sealed class ScheduleClock(DateTimeOffset start) : TimeProvider
|
||||
{
|
||||
private long _ticks = start.UtcTicks;
|
||||
@@ -43,7 +47,14 @@ internal sealed class TestScheduleHost(Archive? archive) : IScheduleHost
|
||||
|
||||
public bool DeepAnalysisAfterScan { get; set; }
|
||||
|
||||
public ValueTask<Archive?> GetArchiveAsync(CancellationToken cancellationToken) => ValueTask.FromResult(Archive);
|
||||
/// <summary>Awaited before the archive is handed out (a slow drive, a pass the test holds).</summary>
|
||||
public Func<CancellationToken, Task>? BeforeArchive { get; set; }
|
||||
|
||||
public async ValueTask<Archive?> GetArchiveAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (BeforeArchive is { } wait) await wait(cancellationToken);
|
||||
return Archive;
|
||||
}
|
||||
|
||||
public string? Unavailable(string action) => Refused.GetValueOrDefault(action);
|
||||
}
|
||||
@@ -529,6 +540,179 @@ public sealed class SchedulerTests(TestContext context)
|
||||
Assert.True(ArchiveJobJson.Read(job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).DeepAnalysis);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Owner review 2, R10: a system schedule that replaced a timer is seeded due when the timer's work was last done a period
|
||||
/// or more before (the newest index copy a week old: the first start after the upgrade copies at once, as the daily loop
|
||||
/// did); done more recently, or never, it waits for the recurrence's first occurrence. Rows that exist keep their runs.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public async Task ASystemScheduleIsSeededDueWhenItsWorkIsAPeriodOld()
|
||||
{
|
||||
await using var fx = await ArchiveFixture.CreateAsync(context);
|
||||
await using var engine = Engine(fx);
|
||||
var clock = new ScheduleClock(T0);
|
||||
var host = new TestScheduleHost(fx.Archive);
|
||||
var daily = ScheduleRecurrence.Interval(1, RecurrenceUnit.Days);
|
||||
host.Systems.Add(new SystemSchedule("index-backup", "Index backup", ScheduleActions.Backup, daily, LastDone: T0.AddDays(-7)));
|
||||
host.Systems.Add(new SystemSchedule("nightly-backup", "Nightly", ScheduleActions.Backup, ScheduleRecurrence.DailyAt(new TimeOnly(3, 0)), LastDone: T0.AddHours(-23)));
|
||||
host.Systems.Add(new SystemSchedule("new-backup", "New", ScheduleActions.Backup, daily));
|
||||
host.Systems.Add(new SystemSchedule("off-backup", "Off", ScheduleActions.Backup, daily, Enabled: false, LastDone: T0.AddDays(-7)));
|
||||
await using (var first = await StartAsync(fx.Db, engine, host, clock))
|
||||
{
|
||||
var due = (await fx.Db.Schedules.GetBySystemKeyAsync("index-backup", Ct))!;
|
||||
Assert.Equal((ScheduleOutcome.Queued, T0, T0.AddDays(1)), (due.LastOutcome!, due.LastRun!.Value, due.NextRun!.Value));
|
||||
var nightly = (await fx.Db.Schedules.GetBySystemKeyAsync("nightly-backup", Ct))!;
|
||||
Assert.Equal(((string?)null, new DateTimeOffset(2026, 10, 3, 3, 0, 0, TimeSpan.Zero)), (nightly.LastOutcome, nightly.NextRun!.Value));
|
||||
var fresh = (await fx.Db.Schedules.GetBySystemKeyAsync("new-backup", Ct))!;
|
||||
Assert.Equal(((string?)null, T0.AddDays(1)), (fresh.LastOutcome, fresh.NextRun!.Value));
|
||||
var off = (await fx.Db.Schedules.GetBySystemKeyAsync("off-backup", Ct))!;
|
||||
Assert.Equal(((string?)null, (DateTimeOffset?)null), (off.LastOutcome, off.NextRun));
|
||||
}
|
||||
// The next start finds the rows: nothing is due again.
|
||||
clock.Advance(TimeSpan.FromHours(1));
|
||||
await using var second = await StartAsync(fx.Db, engine, host, clock);
|
||||
Assert.Count(1, (await fx.Db.Jobs.ListAsync(cancellationToken: Ct)).Items);
|
||||
}
|
||||
|
||||
/// <summary>The loop itself (no pass by hand): its tick fires a due schedule.</summary>
|
||||
[Test]
|
||||
public async Task TheLoopFiresADueScheduleAtItsTick()
|
||||
{
|
||||
await using var fx = await ArchiveFixture.CreateAsync(context);
|
||||
await using var engine = Engine(fx);
|
||||
var clock = new ScheduleClock(T0);
|
||||
await using var scheduler = new Scheduler(fx.Db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromMilliseconds(100) });
|
||||
await scheduler.StartAsync(new TestScheduleHost(fx.Archive), Ct);
|
||||
var changes = Changes(scheduler);
|
||||
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
var ran = await RanAsync(changes, schedule.Id, null);
|
||||
Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (ran.LastOutcome!, ran.NextRun!.Value));
|
||||
Assert.Count(1, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
|
||||
}
|
||||
|
||||
/// <summary><see cref="Scheduler.Wake"/> makes the loop look now, long before its tick.</summary>
|
||||
[Test]
|
||||
public async Task WakeFiresADueScheduleBeforeTheTick()
|
||||
{
|
||||
await using var fx = await ArchiveFixture.CreateAsync(context);
|
||||
await using var engine = Engine(fx);
|
||||
var clock = new ScheduleClock(T0);
|
||||
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
|
||||
var changes = Changes(scheduler);
|
||||
var schedule = await CreateAsync(scheduler, ScheduleActions.Scan);
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
await Task.Delay(200, Ct);
|
||||
Assert.Null((await scheduler.GetAsync(schedule.Id, Ct))!.LastOutcome, "the tick is an hour away");
|
||||
scheduler.Wake();
|
||||
Assert.Equal(ScheduleOutcome.Queued, (await RanAsync(changes, schedule.Id, null)).LastOutcome);
|
||||
}
|
||||
|
||||
/// <summary>Nothing a pass throws (here a subscriber of <see cref="Scheduler.ScheduleChanged"/>) ends the loop: the next run still fires.</summary>
|
||||
[Test]
|
||||
public async Task TheLoopSurvivesAPassThatThrew()
|
||||
{
|
||||
await using var fx = await ArchiveFixture.CreateAsync(context);
|
||||
await using var engine = Engine(fx);
|
||||
var clock = new ScheduleClock(T0);
|
||||
await using var scheduler = await StartAsync(fx.Db, engine, new TestScheduleHost(fx.Archive), clock);
|
||||
var changes = Changes(scheduler);
|
||||
var thrown = 0;
|
||||
scheduler.ScheduleChanged += dto =>
|
||||
{
|
||||
if (dto.LastOutcome is not null && Interlocked.Exchange(ref thrown, 1) == 0) throw new InvalidOperationException("A subscriber failed");
|
||||
};
|
||||
var schedule = await CreateAsync(scheduler, ScheduleActions.Duplicates);
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
scheduler.Wake();
|
||||
var first = await RanAsync(changes, schedule.Id, null);
|
||||
Assert.Equal(1, Volatile.Read(ref thrown));
|
||||
await fx.Db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: Ct);
|
||||
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
scheduler.Wake();
|
||||
var second = await RanAsync(changes, schedule.Id, first.LastJobId);
|
||||
Assert.Equal((ScheduleOutcome.Queued, T0.AddMinutes(20)), (second.LastOutcome!, second.LastRun!.Value));
|
||||
Assert.Count(2, await fx.Db.Jobs.ListByScheduleAsync(schedule.Id, cancellationToken: Ct));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposing stops the loop during a pass: it waits for a host call that does not watch its token, so the database is
|
||||
/// never used after it returns, and cancels one that does.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public async Task DisposeDuringAPassWaitsForItAndStopsIt()
|
||||
{
|
||||
await using var fx = await ArchiveFixture.CreateAsync(context);
|
||||
await using var engine = Engine(fx);
|
||||
var clock = new ScheduleClock(T0);
|
||||
var host = new TestScheduleHost(fx.Archive);
|
||||
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var scheduler = await StartAsync(fx.Db, engine, host, clock);
|
||||
var disposed = false;
|
||||
try
|
||||
{
|
||||
await CreateAsync(scheduler, ScheduleActions.Scan);
|
||||
var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
host.BeforeArchive = async _ =>
|
||||
{
|
||||
entered.TrySetResult();
|
||||
await gate.Task;
|
||||
};
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
scheduler.Wake();
|
||||
await entered.Task.WaitAsync(Budget, Ct);
|
||||
|
||||
var disposing = scheduler.DisposeAsync().AsTask();
|
||||
disposed = true;
|
||||
await Task.Delay(200, Ct);
|
||||
Assert.False(disposing.IsCompleted, "disposed while a pass was still running");
|
||||
gate.SetResult();
|
||||
await disposing.WaitAsync(Budget, Ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
gate.TrySetResult();
|
||||
if (!disposed) await scheduler.DisposeAsync();
|
||||
}
|
||||
|
||||
// A host call that watches the token ends with it.
|
||||
var again = await StartAsync(fx.Db, engine, host, clock);
|
||||
var waiting = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
host.BeforeArchive = async ct =>
|
||||
{
|
||||
waiting.TrySetResult();
|
||||
await Task.Delay(Timeout.Infinite, ct);
|
||||
};
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
again.Wake();
|
||||
await waiting.Task.WaitAsync(Budget, Ct);
|
||||
await again.DisposeAsync().AsTask().WaitAsync(Budget, Ct);
|
||||
}
|
||||
|
||||
private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10));
|
||||
|
||||
/// <summary>Every <see cref="Scheduler.ScheduleChanged"/> from now on.</summary>
|
||||
private static Channel<ScheduleDto> Changes(Scheduler scheduler)
|
||||
{
|
||||
var changes = Channel.CreateUnbounded<ScheduleDto>();
|
||||
scheduler.ScheduleChanged += dto => changes.Writer.TryWrite(dto);
|
||||
return changes;
|
||||
}
|
||||
|
||||
/// <summary>The first change of <paramref name="id"/> that records a run whose job is not <paramref name="previousJob"/>.</summary>
|
||||
private async Task<ScheduleDto> RanAsync(Channel<ScheduleDto> changes, Guid id, Guid? previousJob)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var dto = await changes.Reader.ReadAsync(Ct).AsTask().WaitAsync(Budget, Ct);
|
||||
if (dto.Id == id && dto.LastOutcome is not null && dto.LastJobId != previousJob) return dto;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<Guid> IngestAsync(ArchiveFixture fx, string key, int seed)
|
||||
{
|
||||
var staged = fx.Stage(ArchiveFixture.Start.AddMinutes(seed), seed);
|
||||
|
||||
Reference in new issue
Block a user