CI / Build and test (push) Successful in 7m0s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 13m28s
CI / PostgreSQL 16 integration (push) Successful in 5m52s
CI / PostgreSQL 17 integration (push) Successful in 4m50s
CI / Docker image (push) Successful in 1m4s
Release / Server image (push) Successful in 1m35s
- Jobs > Scheduled on the desktop and the admin panel. Every timer that queued work is now a visible system schedule: archive scan, upload clean-up, index backup, job-history and audit clean-ups, desktop housekeeping. New actions: verify, thumbnails, proxies, analysis, maintenance, backup, clean-ups, reports. - The scheduler: migration 0007, in-house recurrence, compare-and-set claim, catch-up, scan at start. - Log settings: levels, retention, size parts, total limit, daily or hourly rotation, audit retention, diagnostics share; the server writes Debug/Trace to its files again; logging never fails a job. Upgrade: CHRONOS_LOG_LEVEL set in an existing container locks both levels; migration 0007 is one-way. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
134 lines
5.9 KiB
C#
134 lines
5.9 KiB
C#
using System.Threading.Channels;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Data.Jobs;
|
|
using Chronos.Data.Scheduling;
|
|
using Chronos.Data.Tests.Scheduling;
|
|
using Chronos.Protocol;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Jobs;
|
|
|
|
/// <summary>
|
|
/// The job engine and the scheduler log from their error handling: a log line that cannot be written must change nothing
|
|
/// they do. It once left a failed job running (its log line threw before the state was stored, the cause of the
|
|
/// intermittent JobApiTests failures: the process-wide sink was a stopped test server's, whose log folder was gone) and
|
|
/// killed the worker. Serial: these tests replace the process-wide sink.
|
|
/// </summary>
|
|
[Serial]
|
|
[Category("jobs")]
|
|
public sealed class LoggingFailureTests(TestContext context)
|
|
{
|
|
private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero);
|
|
|
|
private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10));
|
|
|
|
/// <summary>Every line fails as Microsoft.Extensions.Logging fails when a file provider's folder is gone.</summary>
|
|
private sealed class BrokenLogging : IDisposable, ILogSink
|
|
{
|
|
private readonly ILogSink _previous = Log.Sink;
|
|
private readonly LogLevel _level = Log.MinimumLevel;
|
|
private int _lines;
|
|
|
|
public BrokenLogging()
|
|
{
|
|
Log.SetSink(this);
|
|
Log.MinimumLevel = LogLevel.Trace;
|
|
}
|
|
|
|
public int Lines => Volatile.Read(ref _lines);
|
|
|
|
public void Write(LogLevel level, string category, string message, Exception? exception)
|
|
{
|
|
Interlocked.Increment(ref _lines);
|
|
throw new AggregateException("An error occurred while writing to logger(s).", new DirectoryNotFoundException("Could not find a part of the path"));
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
Log.SetSink(_previous);
|
|
Log.MinimumLevel = _level;
|
|
}
|
|
}
|
|
|
|
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
|
|
{
|
|
public JobKind Kind => kind;
|
|
|
|
public Task RunAsync(JobContext context) => run(context);
|
|
}
|
|
|
|
private async Task<ChronosDatabase> OpenAsync() =>
|
|
await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
|
|
|
|
private static async Task<Job> WaitForStateAsync(ChronosDatabase db, Guid id, JobState state)
|
|
{
|
|
var until = DateTime.UtcNow + Budget;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
if (await db.Jobs.GetAsync(id) is { } job && job.State == state) return job;
|
|
await Task.Delay(20);
|
|
}
|
|
Assert.Fail($"Job {id} is {(await db.Jobs.GetAsync(id))?.State}, expected {state}");
|
|
return null!;
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFailedJobIsMarkedFailedAndTheWorkerGoesOnWhenLoggingFails()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
var handler = new CallbackHandler(JobKind.Verify, c => c.Job.TargetLabel == "broken" ? throw new InvalidDataException("unreadable") : Task.CompletedTask);
|
|
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
|
|
await engine.StartAsync();
|
|
using var logging = new BrokenLogging();
|
|
|
|
// The engine logs the failure before it stores it: that line failing once left the job running.
|
|
var broken = await engine.EnqueueAsync(JobKind.Verify, [], "broken");
|
|
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed);
|
|
Assert.Equal("unreadable", failed.Error);
|
|
Assert.Greater(logging.Lines, 0);
|
|
|
|
// The one worker is still claiming jobs.
|
|
var next = await engine.EnqueueAsync(JobKind.Verify, [], "fine");
|
|
await WaitForStateAsync(db, next.Id, JobState.Done);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheSchedulerFiresAndRecordsRunsWhenLoggingFails()
|
|
{
|
|
await using var db = await OpenAsync();
|
|
// Never started: the queued backups stay queued.
|
|
await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "t" });
|
|
var clock = new ScheduleClock(T0);
|
|
// Disposed at the end: a loop that faulted would throw there.
|
|
await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) });
|
|
await scheduler.StartAsync(new TestScheduleHost(null), context.Cancellation);
|
|
var runs = Channel.CreateUnbounded<ScheduleDto>();
|
|
scheduler.ScheduleChanged += dto => runs.Writer.TryWrite(dto);
|
|
var schedule = Assert.NotNull((await scheduler.CreateAsync(new CreateScheduleRequest("Backup", ScheduleActions.Backup,
|
|
new RecurrenceDto("every", Every: 10, Unit: "minutes")), null, context.Cancellation)).Schedule);
|
|
// From here every line the loop logs (the claim, the outcome, a failed pass) fails.
|
|
using var logging = new BrokenLogging();
|
|
|
|
async Task<ScheduleDto> RunAsync(Guid? previousJob)
|
|
{
|
|
clock.Advance(TimeSpan.FromMinutes(10));
|
|
scheduler.Wake();
|
|
while (true)
|
|
{
|
|
var dto = await runs.Reader.ReadAsync(context.Cancellation).AsTask().WaitAsync(Budget);
|
|
if (dto.Id == schedule.Id && dto.LastOutcome == ScheduleOutcome.Queued && dto.LastJobId != previousJob) return dto;
|
|
}
|
|
}
|
|
|
|
var first = await RunAsync(null);
|
|
Assert.Equal(clock.Now, first.LastRun);
|
|
await db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: context.Cancellation);
|
|
// The loop survived the lines it could not write: the next run fires too.
|
|
var second = await RunAsync(first.LastJobId);
|
|
Assert.Equal(clock.Now, second.LastRun);
|
|
Assert.Greater(logging.Lines, 0);
|
|
}
|
|
}
|