A desktop sharing the archive that imports into an existing sequence lists the new frames in the sidecar itself and keeps its ThumbsBuilt and ProxiesBuilt. The server scan cleared the flags only when SequenceAdopter.Reconcile found unindexed photos, so it queued no image pass: the new frames, and every frame they renumbered, answered 404 on /thumb and /proxy until a pass was forced. A scan now also clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt when the sidecar's frames version differs from the index row's, decided under the sequence lease, and its follow-ups queue the light analysis and both passes. Writers that change only part of a sequence wrote the whole index row back from a sidecar read under a lease they had already released (decode tiers) or without any (verify stamp): an ingest or a PATCH committed in between was undone, and a row written from a sidecar the index had not synced claimed its frames version, hiding the change from the scan. FrameTiers.RecordAsync and ArchiveVerifier now write the sidecar and only their own columns (SequenceRepository.RecordTiersAsync, RecordVerifiedAsync) under one lease. They and the image passes record the folder's scan stamps only while the index holds the sidecar's frames version (ArchiveIndexer.RecordStampsInStepAsync), so the next scan still sees frames another process added. An image pass decided ThumbsBuilt/ProxiesBuilt from the sidecar it read at its start and set the flag again after a scan had cleared it for photos added meanwhile, so no pass was queued for them. MarkAsync now holds the lease for the sidecar and the index, and sets the flag only while the sidecar lists the photos the pass read at the same indexes; otherwise it clears it. A rebuild deleted every row whose folder was not among those listed when it began, so a sequence that got its first frame or was created by an upload session while it ran lost its row (404 for the Uploader, a second '(2)' sequence for the session). A row is now checked again under its lease, as it and its folder are then, before it goes. Tests: a desktop's ArchiveIngest commit followed by a server scan (Data and Server, /thumb and /proxy answer 200 for every frame); a thumbnail pass with a scan's reconcile in between; tier records and a verify stamp interleaved with a PATCH and with another writer's frames, on SQLite and PostgreSQL; a rebuild during which uploads place and create sequences. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
178 lines
8.3 KiB
C#
178 lines
8.3 KiB
C#
using System.Security.Cryptography;
|
|
using Titano.Core.Archiving;
|
|
using Titano.Core.Codecs.Jpeg;
|
|
using Titano.Core.Imaging;
|
|
using Titano.Core.Metadata;
|
|
using Titano.Core.Model;
|
|
using Titano.Data.Archiving;
|
|
using Titano.Data.Jobs;
|
|
using Titano.Testing;
|
|
|
|
namespace Titano.Data.Tests.Archiving;
|
|
|
|
/// <summary>A temporary archive root with its SQLite index and the archive service.</summary>
|
|
internal sealed class ArchiveFixture : IAsyncDisposable
|
|
{
|
|
public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
|
|
|
|
private readonly TestContext _context;
|
|
private readonly List<IAsyncDisposable> _owned = [];
|
|
|
|
private ArchiveFixture(TestContext context, TitanoDatabase db, ArchiveService service, Archive archive)
|
|
{
|
|
_context = context;
|
|
Db = db;
|
|
Service = service;
|
|
Archive = archive;
|
|
_owned.Add(db);
|
|
}
|
|
|
|
public TitanoDatabase Db { get; private set; }
|
|
public ArchiveService Service { get; private set; }
|
|
public Archive Archive { get; private set; }
|
|
public ArchivePaths Paths => new(Archive.RootPath);
|
|
public CancellationToken Ct => _context.Cancellation;
|
|
|
|
/// <param name="database">The index to use (a PostgreSQL schema); a temporary SQLite file when null.</param>
|
|
public static async Task<ArchiveFixture> CreateAsync(TestContext context, string? root = null, DatabaseOptions? database = null)
|
|
{
|
|
var db = await TitanoDatabase.OpenAsync(database ?? DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation);
|
|
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation);
|
|
return new ArchiveFixture(context, db, service, archive);
|
|
}
|
|
|
|
/// <summary>Switches to a brand-new empty index of the same archive (the "lost database" case).</summary>
|
|
public async Task ReopenWithEmptyIndexAsync()
|
|
{
|
|
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "titano.db")), Ct);
|
|
_owned.Add(db);
|
|
Db = db;
|
|
Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
Archive = await Service.OpenAsync(Archive.RootPath, Ct);
|
|
}
|
|
|
|
public Task<ScanSummary> ScanAsync(ScanMode mode = ScanMode.Incremental) =>
|
|
Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct);
|
|
|
|
/// <summary>
|
|
/// Another process writing the same archive, such as a desktop sharing a server's archive folder: its own index
|
|
/// (scanned) and archive service; disposed with the fixture.
|
|
/// </summary>
|
|
public async Task<(ArchiveService Service, Archive Archive)> OtherWriterAsync()
|
|
{
|
|
var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("other"), "titano.db")), Ct);
|
|
_owned.Add(db);
|
|
var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "other", LeaseWait = TimeSpan.FromSeconds(20) });
|
|
var archive = await service.OpenAsync(Archive.RootPath, Ct);
|
|
await service.Indexer.ScanAsync(archive, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, Ct);
|
|
return (service, archive);
|
|
}
|
|
|
|
/// <summary>A job engine with the archive handlers, started; disposed with the fixture.</summary>
|
|
public async Task<JobEngine> StartEngineAsync()
|
|
{
|
|
var engine = new JobEngine(Db, Service.CreateJobHandlers(),
|
|
new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) });
|
|
_owned.Insert(0, engine);
|
|
await engine.StartAsync(Ct);
|
|
return engine;
|
|
}
|
|
|
|
public async Task<Job> WaitForAsync(Guid jobId, JobState state, int seconds = 30)
|
|
{
|
|
var until = DateTime.UtcNow.AddSeconds(seconds);
|
|
Job? job = null;
|
|
while (DateTime.UtcNow < until)
|
|
{
|
|
job = await Db.Jobs.GetAsync(jobId, Ct);
|
|
if (job?.State == state) return job;
|
|
if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}");
|
|
await Task.Delay(20, Ct);
|
|
}
|
|
Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}");
|
|
return null!;
|
|
}
|
|
|
|
/// <summary>Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it.</summary>
|
|
public async Task<Job> EnqueueWithCheckpointAsync(JobEngine engine, Titano.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint, Guid? id = null)
|
|
{
|
|
var job = new Job
|
|
{
|
|
Id = id ?? Guid.CreateVersion7(),
|
|
Kind = request.Kind,
|
|
TargetIds = request.Targets,
|
|
ParametersJson = request.ParametersJson,
|
|
State = JobState.Queued,
|
|
Checkpoint = checkpoint,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await Db.Jobs.EnqueueAsync(job, Ct);
|
|
engine.Wake();
|
|
return job;
|
|
}
|
|
|
|
/// <summary>Stages a verified upload-like file for ingest: returns its path, size and SHA-256.</summary>
|
|
public (string Path, long Size, string Sha256) Stage(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
|
|
{
|
|
var staging = Path.Combine(Paths.StagingDirectory, "uploads");
|
|
Directory.CreateDirectory(staging);
|
|
var bytes = Photos.Jpeg(captured, seed, make, model);
|
|
var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part");
|
|
File.WriteAllBytes(path, bytes);
|
|
return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes)));
|
|
}
|
|
|
|
public async ValueTask DisposeAsync()
|
|
{
|
|
foreach (var owned in _owned) await owned.DisposeAsync();
|
|
}
|
|
}
|
|
|
|
/// <summary>Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes.</summary>
|
|
internal static class Photos
|
|
{
|
|
public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5")
|
|
{
|
|
using var image = new ImageBuffer(48, 32);
|
|
var random = new Random(seed);
|
|
for (var c = 0; c < 3; c++)
|
|
{
|
|
var plane = image.Plane(c);
|
|
var level = (float)random.NextDouble();
|
|
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f);
|
|
}
|
|
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, BodySerial = "0431", CapturedAt = captured, Iso = 100 });
|
|
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
|
}
|
|
|
|
public static List<string> WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1,
|
|
string make = "Canon", string model = "Canon EOS R5")
|
|
{
|
|
Directory.CreateDirectory(directory);
|
|
var paths = new List<string>(count);
|
|
for (var i = 0; i < count; i++)
|
|
{
|
|
var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG");
|
|
File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model));
|
|
paths.Add(path);
|
|
}
|
|
return paths;
|
|
}
|
|
|
|
/// <summary>The index as comparable text: sequences and frames ordered by folder and index.</summary>
|
|
public static async Task<string> DumpIndexAsync(TitanoDatabase db, Guid archiveId)
|
|
{
|
|
var lines = new List<string>();
|
|
foreach (var location in await db.Sequences.ListLocationsAsync(archiveId))
|
|
{
|
|
var s = (await db.Sequences.GetAsync(location.Id))!;
|
|
lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " +
|
|
$"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}");
|
|
foreach (var f in await db.Frames.ListBySequenceAsync(s.Id))
|
|
lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}");
|
|
}
|
|
return string.Join('\n', lines);
|
|
}
|
|
}
|