Files
Alby96andClaude Opus 5.5 af9eeaab9b
CI / Build and test (push) Successful in 7m0s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 6s
Release / Build, test and publish (push) Successful in 13m28s
CI / PostgreSQL 16 integration (push) Successful in 5m52s
CI / PostgreSQL 17 integration (push) Successful in 4m50s
CI / Docker image (push) Successful in 1m4s
Release / Server image (push) Successful in 1m35s
feat: owner review 2 phase 2: scheduled jobs and log settings (#10)
- Jobs > Scheduled on the desktop and the admin panel. Every timer that queued work is now a visible system schedule: archive scan, upload clean-up, index backup, job-history and audit clean-ups, desktop housekeeping. New actions: verify, thumbnails, proxies, analysis, maintenance, backup, clean-ups, reports.
- The scheduler: migration 0007, in-house recurrence, compare-and-set claim, catch-up, scan at start.
- Log settings: levels, retention, size parts, total limit, daily or hourly rotation, audit retention, diagnostics share; the server writes Debug/Trace to its files again; logging never fails a job.

Upgrade: CHRONOS_LOG_LEVEL set in an existing container locks both levels; migration 0007 is one-way.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-27 11:19:26 +01:00

134 lines
5.9 KiB
C#

using System.Threading.Channels;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Data.Archiving;
using Chronos.Data.Jobs;
using Chronos.Data.Scheduling;
using Chronos.Data.Tests.Scheduling;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Jobs;
/// <summary>
/// The job engine and the scheduler log from their error handling: a log line that cannot be written must change nothing
/// they do. It once left a failed job running (its log line threw before the state was stored, the cause of the
/// intermittent JobApiTests failures: the process-wide sink was a stopped test server's, whose log folder was gone) and
/// killed the worker. Serial: these tests replace the process-wide sink.
/// </summary>
[Serial]
[Category("jobs")]
public sealed class LoggingFailureTests(TestContext context)
{
private static readonly DateTimeOffset T0 = new(2026, 10, 2, 9, 0, 0, TimeSpan.Zero);
private static TimeSpan Budget => TimeSpan.FromSeconds(TestTiming.Budget(10));
/// <summary>Every line fails as Microsoft.Extensions.Logging fails when a file provider's folder is gone.</summary>
private sealed class BrokenLogging : IDisposable, ILogSink
{
private readonly ILogSink _previous = Log.Sink;
private readonly LogLevel _level = Log.MinimumLevel;
private int _lines;
public BrokenLogging()
{
Log.SetSink(this);
Log.MinimumLevel = LogLevel.Trace;
}
public int Lines => Volatile.Read(ref _lines);
public void Write(LogLevel level, string category, string message, Exception? exception)
{
Interlocked.Increment(ref _lines);
throw new AggregateException("An error occurred while writing to logger(s).", new DirectoryNotFoundException("Could not find a part of the path"));
}
public void Dispose()
{
Log.SetSink(_previous);
Log.MinimumLevel = _level;
}
}
private sealed class CallbackHandler(JobKind kind, Func<JobContext, Task> run) : IJobHandler
{
public JobKind Kind => kind;
public Task RunAsync(JobContext context) => run(context);
}
private async Task<ChronosDatabase> OpenAsync() =>
await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory(), "jobs.db")), context.Cancellation);
private static async Task<Job> WaitForStateAsync(ChronosDatabase db, Guid id, JobState state)
{
var until = DateTime.UtcNow + Budget;
while (DateTime.UtcNow < until)
{
if (await db.Jobs.GetAsync(id) is { } job && job.State == state) return job;
await Task.Delay(20);
}
Assert.Fail($"Job {id} is {(await db.Jobs.GetAsync(id))?.State}, expected {state}");
return null!;
}
[Test]
public async Task AFailedJobIsMarkedFailedAndTheWorkerGoesOnWhenLoggingFails()
{
await using var db = await OpenAsync();
var handler = new CallbackHandler(JobKind.Verify, c => c.Job.TargetLabel == "broken" ? throw new InvalidDataException("unreadable") : Task.CompletedTask);
await using var engine = new JobEngine(db, [handler], new JobEngineOptions { Workers = 1, WorkerId = "t", PollInterval = TimeSpan.FromMilliseconds(100) });
await engine.StartAsync();
using var logging = new BrokenLogging();
// The engine logs the failure before it stores it: that line failing once left the job running.
var broken = await engine.EnqueueAsync(JobKind.Verify, [], "broken");
var failed = await WaitForStateAsync(db, broken.Id, JobState.Failed);
Assert.Equal("unreadable", failed.Error);
Assert.Greater(logging.Lines, 0);
// The one worker is still claiming jobs.
var next = await engine.EnqueueAsync(JobKind.Verify, [], "fine");
await WaitForStateAsync(db, next.Id, JobState.Done);
}
[Test]
public async Task TheSchedulerFiresAndRecordsRunsWhenLoggingFails()
{
await using var db = await OpenAsync();
// Never started: the queued backups stay queued.
await using var engine = new JobEngine(db, new ArchiveService(db).CreateJobHandlers(), new JobEngineOptions { WorkerId = "t" });
var clock = new ScheduleClock(T0);
// Disposed at the end: a loop that faulted would throw there.
await using var scheduler = new Scheduler(db, engine, new SchedulerOptions { Time = clock, Tick = TimeSpan.FromHours(1) });
await scheduler.StartAsync(new TestScheduleHost(null), context.Cancellation);
var runs = Channel.CreateUnbounded<ScheduleDto>();
scheduler.ScheduleChanged += dto => runs.Writer.TryWrite(dto);
var schedule = Assert.NotNull((await scheduler.CreateAsync(new CreateScheduleRequest("Backup", ScheduleActions.Backup,
new RecurrenceDto("every", Every: 10, Unit: "minutes")), null, context.Cancellation)).Schedule);
// From here every line the loop logs (the claim, the outcome, a failed pass) fails.
using var logging = new BrokenLogging();
async Task<ScheduleDto> RunAsync(Guid? previousJob)
{
clock.Advance(TimeSpan.FromMinutes(10));
scheduler.Wake();
while (true)
{
var dto = await runs.Reader.ReadAsync(context.Cancellation).AsTask().WaitAsync(Budget);
if (dto.Id == schedule.Id && dto.LastOutcome == ScheduleOutcome.Queued && dto.LastJobId != previousJob) return dto;
}
}
var first = await RunAsync(null);
Assert.Equal(clock.Now, first.LastRun);
await db.Jobs.SetStateAsync(first.LastJobId!.Value, JobState.Done, cancellationToken: context.Cancellation);
// The loop survived the lines it could not write: the next run fires too.
var second = await RunAsync(first.LastJobId);
Assert.Equal(clock.Now, second.LastRun);
Assert.Greater(logging.Lines, 0);
}
}