The kept-plan stamp left out whether a planned sequence's photos are hashed. A verify writes frames.sha256 (and only status and verified_at on the row) without another frames version, so after verifying never-hashed photos the same dry run kept answering "verify the sequence first" for up to five minutes, and its confirmation queued a job certain to fail. - SequenceRepository.GetStampAsync adds, for every planned sequence, how many of its photos have their hash in the index (one grouped query over the archive with all, the selected ids in batches of 500 otherwise). - ReorganizePlanCache keeps a refusal (nested_target, not_found) as it keeps a plan: for the same stamp and templates it is the same, and a long plan that ends refused after a cut-off is no longer planned again on every retry. Failures are still not kept. - Planning is bounded: at most two plans are made at once, the others wait for their turn; a plan still being made 15 minutes after it was asked, let go at capacity, or when the server stops is cancelled, and whoever waits for it gets 503 plan_dropped. A read that hangs keeps its turn until it returns but no longer keeps its waiters. - The Capacity summary now says what eviction does (the kept plan asked for first, else the plan being made asked for first); the stamp, protocol and endpoint docs say that a verify hashing photos is a change the plan reads. - Tests: a scan's photos, verified, are planned again and renamed by the job; the stamp follows hashes of planned photos only; refusals kept, failures not; turns, eviction and deadline of plans being made, a hung read included. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 56582df45453fc555e525f19de7de1803204d12c)
114 lines
6.8 KiB
C#
114 lines
6.8 KiB
C#
using Chronos.Core.Model;
|
|
using Chronos.Data.Repositories;
|
|
using Chronos.Data.Tests.Infrastructure;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Data.Tests.Repositories;
|
|
|
|
/// <summary>
|
|
/// The stamp of what a Reorganize plans from in an archive's index (<see cref="SequenceRepository.GetStampAsync"/>) on both
|
|
/// dialects: the server makes a kept Reorganize plan again once it is another one, and only then (owner review 2, R7).
|
|
/// </summary>
|
|
public abstract class SequenceStampTests(TestContext context) : RepositoryTest(context)
|
|
{
|
|
[Test]
|
|
public async Task TheStampFollowsWhatAReorganizePlansFromAndNothingElse()
|
|
{
|
|
var db = await OpenChronosAsync();
|
|
var archive = TestData.Archive();
|
|
var other = TestData.Archive("Other");
|
|
await db.Archives.UpsertAsync(archive, Ct);
|
|
await db.Archives.UpsertAsync(other, Ct);
|
|
var roof = TestData.Sequence(archive.Id, new DateTime(2026, 9, 28, 18, 0, 0), "Roof");
|
|
var inner = TestData.Sequence(archive.Id, null, "Inner") with { Folder = roof.Folder + "/inner" };
|
|
var ridge = TestData.Sequence(archive.Id, new DateTime(2026, 9, 29, 7, 0, 0), "Ridge");
|
|
IReadOnlyList<Guid> planned = [roof.Id];
|
|
var empty = await db.Sequences.GetStampAsync(archive.Id, planned, Ct);
|
|
foreach (var sequence in new[] { roof, inner, ridge }) await db.Sequences.UpsertAsync(sequence, Ct);
|
|
var stamp = await db.Sequences.GetStampAsync(archive.Id, planned, Ct);
|
|
Assert.NotEqual(empty, stamp);
|
|
Assert.Equal(stamp, await db.Sequences.GetStampAsync(archive.Id, planned, Ct));
|
|
Assert.NotEqual(stamp, await db.Sequences.GetStampAsync(archive.Id, null, Ct));
|
|
|
|
// What the plan does not read: another archive, and the writes of every upload, verify, tag or status change elsewhere.
|
|
await db.Sequences.UpsertAsync(TestData.Sequence(other.Id), Ct);
|
|
await VerifyAsync(db, await PhotoAsync(db, ridge, 0, hashed: false));
|
|
await VerifyAsync(db, await PhotoAsync(db, inner, 0, hashed: false));
|
|
Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(ridge.Id, new SequenceEdit { Title = "Ridge at dawn", Notes = "cold" }, null, Ct)).Outcome);
|
|
foreach (var id in new[] { roof.Id, ridge.Id })
|
|
{
|
|
Assert.True(await db.Sequences.SetTagsAsync(id, ["sky"], Ct));
|
|
Assert.True(await db.Sequences.UpdateStatusAsync(id, SequenceStatus.ProxiesBuilt, SequenceStatus.None, Ct));
|
|
Assert.True(await db.Sequences.RecordVerifiedAsync(id, SequenceStatus.None, SequenceStatus.None, TestData.Now, Ct));
|
|
}
|
|
Assert.Equal(stamp, await db.Sequences.GetStampAsync(archive.Id, planned, Ct));
|
|
|
|
// What it reads gives one never seen before: the sequences planned, the titles inside their folders, every folder.
|
|
var seen = new HashSet<string>(StringComparer.Ordinal) { empty, stamp };
|
|
async Task ChangedAsync(string what) => Assert.True(seen.Add(await db.Sequences.GetStampAsync(archive.Id, planned, Ct)), what);
|
|
Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(roof.Id, new SequenceEdit { Title = "Roof at noon" }, null, Ct)).Outcome);
|
|
await ChangedAsync("title");
|
|
var scanned = await PhotoAsync(db, roof, 0, hashed: false);
|
|
await ChangedAsync("photos");
|
|
await VerifyAsync(db, scanned);
|
|
await ChangedAsync("photos hashed by a verify, which a file template renames");
|
|
Assert.Equal(SequenceEditOutcome.Updated, (await db.Sequences.EditAsync(inner.Id, new SequenceEdit { Title = "Inner court" }, null, Ct)).Outcome);
|
|
await ChangedAsync("title of a sequence a moving folder would carry");
|
|
Assert.True(await db.Sequences.RelocateAsync(ridge.Id, "2026/ridge", null, Ct));
|
|
await ChangedAsync("folder of a sequence not planned");
|
|
Assert.True(await db.Sequences.RelocateAsync(roof.Id, "2026/moved", null, Ct));
|
|
await ChangedAsync("folder");
|
|
var added = TestData.Sequence(archive.Id);
|
|
await db.Sequences.UpsertAsync(added, Ct);
|
|
await ChangedAsync("a sequence added");
|
|
Assert.True(await db.Sequences.DeleteAsync(added.Id, Ct));
|
|
Assert.Contains(await db.Sequences.GetStampAsync(archive.Id, planned, Ct), seen);
|
|
|
|
// With every sequence planned, a photo anywhere changes the plan read, and so does a verify that hashes one; a verify
|
|
// that hashes none still does not.
|
|
var all = await db.Sequences.GetStampAsync(archive.Id, null, Ct);
|
|
Assert.True(await db.Sequences.RecordVerifiedAsync(ridge.Id, SequenceStatus.None, SequenceStatus.None, TestData.Now.AddDays(1), Ct));
|
|
Assert.Equal(all, await db.Sequences.GetStampAsync(archive.Id, null, Ct));
|
|
var photo = await PhotoAsync(db, ridge, 1, hashed: false);
|
|
var uploaded = await db.Sequences.GetStampAsync(archive.Id, null, Ct);
|
|
Assert.NotEqual(all, uploaded);
|
|
await VerifyAsync(db, photo);
|
|
var verified = await db.Sequences.GetStampAsync(archive.Id, null, Ct);
|
|
Assert.False(verified == all || verified == uploaded, "a photo hashed");
|
|
}
|
|
|
|
/// <summary>
|
|
/// A photo indexed into a sequence, as an upload does, or without its hash, as a scan does: the frame, then the row's
|
|
/// counts and frames version.
|
|
/// </summary>
|
|
private async Task<Frame> PhotoAsync(ChronosDatabase db, Sequence sequence, int index, bool hashed = true)
|
|
{
|
|
var frame = TestData.Frame(sequence.Id, index);
|
|
frame = frame with { RelativePath = $"{sequence.Folder}/originals/{frame.FileName}", Sha256 = hashed ? frame.Sha256 : null };
|
|
await db.Frames.BulkInsertAsync([frame], Ct);
|
|
Assert.NotNull(await db.Sequences.RefreshStatsAsync(sequence.Id, Ct));
|
|
return frame;
|
|
}
|
|
|
|
/// <summary>What a verify writes once it hashed a photo never hashed: the hash, then the sequence's verify time, and not another frames version.</summary>
|
|
private async Task VerifyAsync(ChronosDatabase db, Frame frame)
|
|
{
|
|
var version = (await db.Sequences.GetAsync(frame.SequenceId, Ct))!.FramesVersion;
|
|
var sha256 = TestData.Frame(frame.SequenceId, frame.Index).Sha256;
|
|
Assert.Equal(1, await db.Frames.UpdateIntegrityAsync([new FrameIntegrity(frame.Id, FrameFlags.None, sha256)], Ct));
|
|
Assert.True(await db.Sequences.RecordVerifiedAsync(frame.SequenceId, SequenceStatus.Verified, SequenceStatus.None, TestData.Now, Ct));
|
|
Assert.Equal(version, (await db.Sequences.GetAsync(frame.SequenceId, Ct))!.FramesVersion);
|
|
}
|
|
}
|
|
|
|
public sealed class SqliteSequenceStampTests(TestContext context) : SequenceStampTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.SqliteMemory;
|
|
}
|
|
|
|
[RequiresEnvironment(PgTestServer.Variable)]
|
|
public sealed class PostgresSequenceStampTests(TestContext context) : SequenceStampTests(context)
|
|
{
|
|
protected override Backend Backend => Backend.Postgres;
|
|
}
|