Files
Chronos/tests/Chronos.Data.Tests/Sqlite/DatabaseBackupsTests.cs
T
Alby96 e384564784 fix: logging never fails a job, one backup retention, first copy after the upgrade
Owner review 2, phase 2 review findings (scheduler core).

- Log.Write caught only IOException. Microsoft.Extensions.Logging wraps
  provider failures in an AggregateException, so a log line that could not
  be written (in tests: the process-wide sink of a stopped server whose log
  folder was gone) escaped JobEngine.RunAsync's catch before the failure was
  stored: the job stayed running and the worker faulted. This was the cause
  of the intermittent JobApiTests/ArchiveModuleTests failures. Log.Write now
  drops any non-fatal sink failure, TeeLogSink keeps a line from the other
  sinks, FileLogSink drops a line it cannot write and reopens the file at the
  next one, and LoggerSink drops what the providers throw. The scheduler's
  loop relies on the same guarantee.
- The daily backup loop (RunDailyAsync, IsDue, Interval, CheckInterval) had
  no caller since the index-backup schedule replaced it; it is removed and
  the DatabaseBackups summary describes the schedule.
- The first copy after the upgrade no longer waits a day (or for 03:00):
  SystemSchedule.LastDone seeds a new row due at once when its work was last
  done a period or more before, and both hosts pass the newest copy in the
  backups folder for index-backup.
- Back up now (Settings and Archive), the copy kept before a restore and a
  backup job that keeps fewer prune to the largest Copies kept of the backup
  schedules (DatabaseBackups.KeptAsync), not a fixed three. The four
  tooltips that promised three daily copies are reworded (en, it) and their
  stale translations deleted.
- New tests for the scheduler loop itself: a tick, Wake, a pass that threw,
  and dispose during a pass.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit c5ad0c4c3f636129afe5dd1ce039185516da5621)
2026-10-27 09:56:47 +01:00

261 lines
16 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Chronos.Core.Model;
using Chronos.Data.Repositories;
using Chronos.Data.Scheduling;
using Chronos.Protocol;
using Chronos.Testing;
namespace Chronos.Data.Tests.Sqlite;
/// <summary>
/// BRIEF §6.2: VACUUM INTO copies of the SQLite index (the index backup schedule's, Back up now's, the one kept before a
/// restore), as many kept as the backup schedules keep, and a restore from Settings › Advanced that waits for the next
/// start, runs before the index opens (so it recovers one that no longer opens), checks the copy first (quick_check
/// included) and never replaces a working index with a broken one.
/// </summary>
[Category("sqlite")]
public sealed class DatabaseBackupsTests(TestContext context)
{
private async Task<(ChronosDatabase Db, string Path)> OpenAsync(string name = "chronos.db")
{
var path = System.IO.Path.Combine(context.CreateTempDirectory("db"), name);
return (await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), context.Cancellation), path);
}
private static Task AddArchiveAsync(ChronosDatabase db, string name) =>
db.Archives.UpsertAsync(new Archive { Id = Guid.CreateVersion7(), Name = name, RootPath = "/archives/" + name, Created = DateTimeOffset.UtcNow });
[Test]
public async Task ListsTheCopiesOfThisIndexNewestFirst()
{
var directory = context.CreateTempDirectory("backups");
File.WriteAllText(Path.Combine(directory, "chronos-20261010-030000-000.db"), "a");
File.WriteAllText(Path.Combine(directory, "chronos-20261012-030000-000.db"), "bb");
File.WriteAllText(Path.Combine(directory, "chronos-20261011-030000-000.db"), "ccc");
File.WriteAllText(Path.Combine(directory, "other-20261013-030000-000.db"), "x");
File.WriteAllText(Path.Combine(directory, "chronos-notes.db"), "x");
var backups = DatabaseBackups.List(directory, "/data/chronos.db");
Assert.SequenceEqual(["chronos-20261012-030000-000.db", "chronos-20261011-030000-000.db", "chronos-20261010-030000-000.db"],
backups.Select(b => Path.GetFileName(b.Path)));
Assert.Equal(new DateTimeOffset(2026, 10, 12, 3, 0, 0, TimeSpan.Zero), backups[0].Created);
Assert.Equal(2L, backups[0].Bytes);
Assert.Empty(DatabaseBackups.List(Path.Combine(directory, "missing"), "/data/chronos.db"));
}
[Test]
public async Task TheNewestCopyIsWhatTheIndexBackupIsSeededWith()
{
var (db, path) = await OpenAsync();
await using var _ = db;
var directory = context.CreateTempDirectory("backups");
Assert.Null(DatabaseBackups.Newest(db, directory));
File.WriteAllText(Path.Combine(directory, "chronos-20261010-030000-000.db"), "a");
File.WriteAllText(Path.Combine(directory, "chronos-20261012-030000-000.db"), "b");
File.WriteAllText(Path.Combine(directory, "other-20261013-030000-000.db"), "x");
Assert.Equal(new DateTimeOffset(2026, 10, 12, 3, 0, 0, TimeSpan.Zero), DatabaseBackups.Newest(db, directory));
// Without a folder of its own, the one next to the index.
Assert.Null(DatabaseBackups.Newest(db, null));
await db.BackupAsync(DatabaseBackups.DefaultDirectory(path), DatabaseBackups.Keep, context.Cancellation);
Assert.NotNull(DatabaseBackups.Newest(db, null));
await using var memory = await ChronosDatabase.OpenAsync(DatabaseOptions.SqliteInMemory(), context.Cancellation);
Assert.Null(DatabaseBackups.Newest(memory, directory));
}
/// <summary>
/// One folder, one retention (owner review 2, R10): Back up now and the copy kept before a restore keep as many copies
/// as the backup schedules keep (the largest Copies kept, enabled or not), not a fixed three.
/// </summary>
[Test]
public async Task EveryCopyKeepsAsManyAsTheBackupSchedulesKeep()
{
var (db, path) = await OpenAsync();
var directory = context.CreateTempDirectory("backups");
Assert.Equal(DatabaseBackups.Keep, await DatabaseBackups.KeptAsync(db, context.Cancellation));
await db.Schedules.EnsureSystemAsync(
[
new SystemSchedule(DatabaseBackups.ScheduleKey, "Index backup", ScheduleActions.Backup, ScheduleRecurrence.Parse("every:1d"), new ScheduleParametersDto(Keep: 5)),
new SystemSchedule("other-backup", "Other backup", ScheduleActions.Backup, ScheduleRecurrence.Parse("every:1d"), new ScheduleParametersDto(Keep: 1), Enabled: false),
new SystemSchedule("cleanup", "Clean-up", ScheduleActions.Cleanup, ScheduleRecurrence.Parse("daily:04:00"), new ScheduleParametersDto(Steps: ["audit"], OlderThanDays: 30)),
], DateTimeOffset.UtcNow, context.Cancellation);
Assert.Equal(5, await DatabaseBackups.KeptAsync(db, context.Cancellation));
// Five copies of the schedule, then Back up now: the four newest stay with the new one.
for (var i = 0; i < 5; i++) await db.BackupAsync(directory, 5, context.Cancellation);
var scheduled = DatabaseBackups.List(directory, path).Select(b => b.Path).ToList();
var manual = await db.BackupAsync(directory, await DatabaseBackups.KeptAsync(db, context.Cancellation), context.Cancellation);
Assert.SequenceEqual([manual, .. scheduled.Take(4)], DatabaseBackups.List(directory, path).Select(b => b.Path));
// The copy of the index a restore replaces prunes the same way.
await DatabaseBackups.StageRestoreAsync(scheduled[0], path, context.Cancellation);
await db.DisposeAsync();
Assert.True(await DatabaseBackups.RestoreStagedAsync(path, directory, context.Cancellation));
var after = DatabaseBackups.List(directory, path).Select(b => b.Path).ToList();
Assert.Count(5, after);
Assert.SequenceEqual([manual, .. scheduled.Take(3)], after.Skip(1));
}
[Test]
public async Task AStagedRestoreReplacesTheIndexAtTheNextOpen()
{
var (db, path) = await OpenAsync();
var directory = context.CreateTempDirectory("backups");
await AddArchiveAsync(db, "kept");
var backup = await db.BackupAsync(directory, DatabaseBackups.Keep, context.Cancellation);
await AddArchiveAsync(db, "added later");
Assert.Equal(new DatabaseSummary(db.SchemaVersion, 2, 0, 0), await DatabaseBackups.SummarizeAsync(db, context.Cancellation));
// The dry run reads the copy without touching it.
var bytes = await File.ReadAllBytesAsync(backup, context.Cancellation);
var summary = await DatabaseBackups.SummarizeBackupAsync(backup, context.CreateTempDirectory("scratch"), context.Cancellation);
Assert.Equal(1, summary.Archives);
Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(backup, context.Cancellation));
await DatabaseBackups.StageRestoreAsync(backup, path, context.Cancellation);
Assert.True(DatabaseBackups.HasStagedRestore(path));
await db.DisposeAsync();
// The next start, before the index opens: the index as it was is copied first, then it is the chosen copy and
// the staged file is gone (consumed by the swap, so it can never be applied twice).
Assert.True(await DatabaseBackups.RestoreStagedAsync(path, directory, context.Cancellation));
await using var reopened = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), context.Cancellation);
Assert.SequenceEqual(["kept"], (await reopened.Archives.ListAsync(context.Cancellation)).Select(a => a.Name));
Assert.False(DatabaseBackups.HasStagedRestore(path));
var copies = DatabaseBackups.List(directory, path);
Assert.Count(2, copies);
var replaced = await DatabaseBackups.SummarizeBackupAsync(copies[0].Path, context.CreateTempDirectory("scratch"), context.Cancellation);
Assert.Equal(2, replaced.Archives);
Assert.False(await DatabaseBackups.RestoreStagedAsync(path, directory, context.Cancellation), "nothing staged any more");
}
[Test]
public async Task AStagedRestoreRecoversAnIndexThatNoLongerOpens()
{
var (db, path) = await OpenAsync();
var directory = context.CreateTempDirectory("backups");
await AddArchiveAsync(db, "kept");
await db.BackupAsync(directory, DatabaseBackups.Keep, context.Cancellation);
var newest = await db.BackupAsync(directory, DatabaseBackups.Keep, context.Cancellation);
var daily = DatabaseBackups.List(directory, path).Select(b => b.Path).ToList();
await DatabaseBackups.StageRestoreAsync(newest, path, context.Cancellation);
await db.DisposeAsync();
// The index is damaged beyond opening ("file is not a database").
File.Delete(path + "-wal");
File.Delete(path + "-shm");
var garbage = new byte[16384];
new Random(7).NextBytes(garbage);
await File.WriteAllBytesAsync(path, garbage, context.Cancellation);
await Assert.ThrowsAsync<DatabaseException>(async () => await (await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), context.Cancellation)).DisposeAsync());
Assert.True(await DatabaseBackups.RestoreStagedAsync(path, directory, context.Cancellation));
await using var reopened = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), context.Cancellation);
Assert.SequenceEqual(["kept"], (await reopened.Archives.ListAsync(context.Cancellation)).Select(a => a.Name));
Assert.False(DatabaseBackups.HasStagedRestore(path));
// The damaged index is kept whole beside the daily copies, outside their rotation: none of them was pushed out.
Assert.SequenceEqual(daily, DatabaseBackups.List(directory, path).Select(b => b.Path));
var aside = Assert.Contains(Directory.GetFiles(directory), f => Path.GetFileName(f).StartsWith("chronos-replaced-", StringComparison.Ordinal));
Assert.SequenceEqual(garbage, await File.ReadAllBytesAsync(aside, context.Cancellation));
}
[Test]
public async Task ABrokenStagedCopyIsSetAsideAndTheIndexKept()
{
var (db, path) = await OpenAsync();
await AddArchiveAsync(db, "current");
await db.DisposeAsync();
await File.WriteAllTextAsync(DatabaseBackups.StagedPath(path), "this is not a database", context.Cancellation);
var backups = context.CreateTempDirectory("backups");
await Assert.ThrowsAsync<InvalidDataException>(() => DatabaseBackups.RestoreStagedAsync(path, backups, context.Cancellation));
Assert.False(DatabaseBackups.HasStagedRestore(path));
Assert.FileExists(DatabaseBackups.StagedPath(path) + ".failed");
Assert.Empty(Directory.GetFiles(backups));
await using var reopened = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), context.Cancellation);
Assert.SequenceEqual(["current"], (await reopened.Archives.ListAsync(context.Cancellation)).Select(a => a.Name));
await Assert.ThrowsAsync<InvalidDataException>(() =>
DatabaseBackups.SummarizeBackupAsync(DatabaseBackups.StagedPath(path) + ".failed", context.CreateTempDirectory("scratch"), context.Cancellation));
}
[Test]
public async Task AStagedCopyThatFailsItsIntegrityCheckIsSetAsideBeforeAnyCopyIsMade()
{
var (db, path) = await OpenAsync();
var directory = context.CreateTempDirectory("backups");
await AddArchiveAsync(db, "current");
// A table the summary never reads, filled so its pages come last in a VACUUM INTO copy.
await using (var session = await db.Database.OpenAsync(context.Cancellation))
{
await session.ExecuteScriptAsync("""
CREATE TABLE filler (id INTEGER PRIMARY KEY, data BLOB);
WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 200) INSERT INTO filler (data) SELECT randomblob(1000) FROM n;
""", context.Cancellation);
}
for (var i = 0; i < DatabaseBackups.Keep; i++) await db.BackupAsync(directory, DatabaseBackups.Keep, context.Cancellation);
var daily = DatabaseBackups.List(directory, path).Select(b => b.Path).ToList();
await db.DisposeAsync();
// One damaged page in the filler table of the copy: the header, the schema and every counted table are intact.
var damaged = Path.Combine(context.CreateTempDirectory("damaged"), "copy.db");
File.Copy(daily[0], damaged);
var bytes = await File.ReadAllBytesAsync(damaged, context.Cancellation);
var pageSize = (bytes[16] << 8) | bytes[17];
if (pageSize == 1) pageSize = 65536;
bytes[bytes.Length - pageSize] = 0xFF;
await File.WriteAllBytesAsync(damaged, bytes, context.Cancellation);
// Opening, migrating and counting do not notice it: only quick_check does.
var probe = Path.Combine(context.CreateTempDirectory("probe"), "chronos.db");
File.Copy(damaged, probe);
await using (var opened = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(probe), context.Cancellation))
Assert.Equal(1, (await DatabaseBackups.SummarizeAsync(opened, context.Cancellation)).Archives);
await Assert.ThrowsAsync<InvalidDataException>(() => DatabaseBackups.SummarizeBackupAsync(damaged, context.CreateTempDirectory("scratch"), context.Cancellation));
await DatabaseBackups.StageRestoreAsync(damaged, path, context.Cancellation);
await Assert.ThrowsAsync<InvalidDataException>(() => DatabaseBackups.RestoreStagedAsync(path, directory, context.Cancellation));
Assert.False(DatabaseBackups.HasStagedRestore(path), "set aside: the next start neither retries it nor makes another copy");
Assert.FileExists(DatabaseBackups.StagedPath(path) + ".failed");
Assert.SequenceEqual(daily, DatabaseBackups.List(directory, path).Select(b => b.Path));
Assert.False(await DatabaseBackups.RestoreStagedAsync(path, directory, context.Cancellation));
await using var reopened = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), context.Cancellation);
Assert.SequenceEqual(["current"], (await reopened.Archives.ListAsync(context.Cancellation)).Select(a => a.Name));
}
[Test]
public async Task CancellingAStagedRestoreKeepsTheIndex()
{
var (db, path) = await OpenAsync();
var directory = context.CreateTempDirectory("backups");
var backup = await db.BackupAsync(directory, DatabaseBackups.Keep, context.Cancellation);
await DatabaseBackups.StageRestoreAsync(backup, path, context.Cancellation);
await db.DisposeAsync();
DatabaseBackups.CancelStagedRestore(path);
Assert.False(DatabaseBackups.HasStagedRestore(path));
Assert.False(await DatabaseBackups.RestoreStagedAsync(path, directory, context.Cancellation));
Assert.FileExists(backup);
}
[Test]
public async Task ARestoreWaitsWhenTheIndexCannotBeBackedUpFirst()
{
var (db, path) = await OpenAsync();
var backup = await db.BackupAsync(context.CreateTempDirectory("backups"), DatabaseBackups.Keep, context.Cancellation);
await AddArchiveAsync(db, "since the backup");
await DatabaseBackups.StageRestoreAsync(backup, path, context.Cancellation);
await db.DisposeAsync();
// The backups folder cannot be created: a file is in its way.
var blocked = Path.Combine(context.CreateTempDirectory("blocked"), "backups");
await File.WriteAllTextAsync(blocked, "not a folder", context.Cancellation);
await Assert.ThrowsAsync<IOException>(() => DatabaseBackups.RestoreStagedAsync(path, blocked, context.Cancellation));
Assert.True(DatabaseBackups.HasStagedRestore(path), "the restore stays staged for the next start");
await using var reopened = await ChronosDatabase.OpenAsync(DatabaseOptions.Sqlite(path), context.Cancellation);
Assert.SequenceEqual(["since the backup"], (await reopened.Archives.ListAsync(context.Cancellation)).Select(a => a.Name));
}
}