Files
Chronos/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
T
Claude 94ef3c88db fix(archive): partial writers keep the folder changed for the scan; image flags follow queued passes and hashed photos
Writers that change part of a sequence (decode tiers, image-pass flags,
analyses, verify flushes and stamps, duplicate flags, title/tags/rating/
notes edits, mass rename) now ask ArchiveIndexer.InStepAsync before their
own sidecar write and record the folder's scan stamps afterwards only when
the index held the folder as it was: the stored stamps are still the
folder's and the row holds the sidecar's frames. What another process
changed in the sidecar alone (frames, a new title) stays visible to the
next incremental scan, which writes the whole row from the sidecar.

EditSequenceInfoAsync reads sidecar and row under the lease and writes the
row through SequenceRepository.EditAsync at the settings version it read,
applying the edit again on a conflict, then the sidecar from the stored
row; settings and frames committed meanwhile stay. MassRenamer writes only
folder and title (RelocateAsync, whose null title now keeps the current
one). Verify flushes and duplicate flags run under one lease.

A scan that finds frames another writer listed in the sidecar itself
clears AnalyzedLight, ThumbsBuilt and ProxiesBuilt only in its own row and
only on a server (ImagesAfterScan); the shared sidecar keeps the writer's
flags, so a desktop's poll no longer makes the server redo its passes.

An image pass overtaken by an upload keeps its flag while a pass of the
same kind is queued for the sequence. ThumbPack.IsFrom accepts a thumbnail
drawn before a verify hashed its photo, in the pass, in SameFrames and in
the server's thumbnail lookup, which answered 404 for every such frame.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 20:16:58 +00:00

403 lines
23 KiB
C#

using System.Net;
using System.Security.Cryptography;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Archiving;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
using Titano.Server.Storage;
using Titano.Testing;
using UploadState = Titano.Data.Repositories.UploadState;
namespace Titano.Server.Tests.Storage;
/// <summary>The production composition: uploads committed by the real archive engine, thumbnails built by its job.</summary>
[Category("server")]
public sealed class ArchiveModuleTests(TestContext context)
{
[Test]
public async Task UploadsLandInTheArchiveLayoutAndGetThumbnails()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo();
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
var sequence = Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId));
var paths = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder);
Assert.True(File.Exists(paths.ManifestPath), "sequence.json");
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, row.RelativePath)));
Assert.Count(0, Directory.GetFiles(Path.Combine(server.Options.ArchivePath, ".titano", "staging")));
// The commit queues a thumbnail job; its handler comes from the archive module.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (true)
{
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.FrameId}/thumb");
if (thumb.StatusCode == HttpStatusCode.OK)
{
var jpeg = await thumb.Content.ReadAsByteArrayAsync();
Assert.True(jpeg is [0xFF, 0xD8, ..], "a JPEG thumbnail");
break;
}
Assert.True(DateTime.UtcNow < until, $"thumbnail not built: {(int)thumb.StatusCode}");
await Task.Delay(100);
}
}
[Test]
public async Task AServerOnSqliteBacksUpItsIndexOnStart()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var backups = Path.Combine(server.Options.DataPath, "backups");
// No copy yet, so the first one is made at once (BRIEF §6.2), named after the index it copies.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (!Directory.Exists(backups) || Directory.GetFiles(backups, "*.db").Length == 0)
{
Assert.True(DateTime.UtcNow < until, "no backup of the index");
await Task.Delay(50);
}
var path = Assert.NotNull(((Titano.Data.Sqlite.SqliteDatabase)server.Database.Database).Path);
var backup = Titano.Data.DatabaseBackups.List(backups, path).Single();
var summary = await Titano.Data.DatabaseBackups.SummarizeBackupAsync(backup.Path, context.CreateTempDirectory("scratch"), context.Cancellation);
Assert.Equal(server.Database.SchemaVersion, summary.SchemaVersion);
}
[Test]
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPollingWithThumbnailsAndProxies()
{
var photo = Photo();
// An adopted folder that already holds photos.
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
// The scan queues the thumbnails and proxies clients load from the server, as an upload does.
await UntilImagesAsync(server, editor, archive.Id, "roof");
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3));
var polling = server.Services.GetRequiredService<ArchivePolling>();
var paused = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Scan,
TargetIds = [archive.Id],
State = JobState.Paused,
Created = DateTimeOffset.UtcNow,
};
await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation);
Assert.Null(await polling.PollAsync(context.Cancellation));
await server.Services.GetRequiredService<JobEngineHost>().Engine.CancelAsync(paused.Id, context.Cancellation);
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
Assert.Equal(JobPriority.Background, scan.Priority);
await UntilIndexedAsync(server, archive.Id, "garden");
await UntilImagesAsync(server, editor, archive.Id, "garden");
}
[Test]
public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll()
{
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo());
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
await UntilImagesAsync(server, editor, archive.Id, "roof");
await UntilIdleAsync(server);
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
// A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files
// the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt.
var desktopFolder = context.CreateTempDirectory("desktop");
await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation))
{
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" });
var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation);
await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation);
var photo = Photo(seed: 5);
var staged = Path.Combine(desktopFolder, "IMG_0001.JPG");
await File.WriteAllBytesAsync(staged, photo);
await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length,
Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation);
}
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
// The next poll indexes it and queues the images: the new frame and the one it moved get theirs.
Assert.NotNull(await server.Services.GetRequiredService<ArchivePolling>().PollAsync(context.Cancellation));
await UntilScannedAsync(server);
Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
await UntilImagesAsync(server, editor, archive.Id, "roof");
}
[Test]
public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed()
{
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo());
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
await UntilImagesAsync(server, editor, archive.Id, "roof");
await UntilIdleAsync(server);
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single();
Assert.Null(frame.Sha256);
// A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own.
var verified = await server.Services.GetRequiredService<ArchiveService>().Verifier.VerifyAsync(
Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null,
context.Cancellation);
Assert.Equal(1L, verified.Hashed);
Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256);
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb");
Assert.Equal(HttpStatusCode.OK, thumb.StatusCode);
Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
}
[Test]
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var polling = server.Services.GetRequiredService<ArchivePolling>();
// Nothing changed: the served archive (the one an administrator's edit put there) is not replaced by a row read again.
var served = archive.Current;
await polling.PollAsync(context.Cancellation);
Assert.True(ReferenceEquals(served, archive.Current), "a poll that registered nothing keeps the served archive");
// A desktop sharing the archive changed its folder template and proxy size in archive.json.
var path = new ArchivePaths(archive.Root).ManifestPath;
var manifest = ManifestStore.ReadArchive(path);
ManifestStore.WriteArchive(path, manifest with
{
LayoutTemplates = manifest.LayoutTemplates with { Folder = "{camera}/{date}" },
ProxyPolicy = manifest.ProxyPolicy with { LongEdge = 2560 },
});
await polling.PollAsync(context.Cancellation);
// Uploads (the index row) and the admin panel (the served archive) follow it.
Assert.Equal(("{camera}/{date}", 2560), (archive.Current.FolderTemplate, archive.Current.ProxyLongEdge));
var row = Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, context.Cancellation));
Assert.Equal(("{camera}/{date}", 2560), (row.FolderTemplate, row.ProxyLongEdge));
// The admin panel opens before the next poll: its form, which saves every field, starts from the desktop's change.
ManifestStore.WriteArchive(path, ManifestStore.ReadArchive(path) with { LayoutTemplates = manifest.LayoutTemplates with { Folder = "{year}/{camera}" } });
var admin = await server.AdminAsync();
using (var read = await admin.GetAsync("/api/v1/archive/settings"))
Assert.Equal("{year}/{camera}", (await read.ReadAsync(ProtocolJson.Default.ArchiveSettingsDto)).FolderTemplate);
Assert.Equal("{year}/{camera}", archive.Current.FolderTemplate);
// A refresh that read the archive before an administrator saved an edit never puts it back over that edit.
var before = archive.Current;
archive.Current = before with { Name = "Edited" };
Assert.False(archive.TryReplace(before, before with { Name = "Stale" }), "the edit saved meanwhile stays");
Assert.Equal("Edited", archive.Current.Name);
}
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder))
{
Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed");
await Task.Delay(50);
}
}
/// <summary>
/// Waits until every frame of the sequence in <paramref name="folder"/> has its thumbnail and proxy served and the
/// passes that made them have marked the sequence (a pass writes its files before it marks).
/// </summary>
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
{
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
{
foreach (var asset in new[] { "thumb", "proxy" })
{
while (true)
{
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
if (response.StatusCode == HttpStatusCode.OK)
{
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}");
break;
}
Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}");
await Task.Delay(100);
}
}
}
while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built
&& !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt)))
{
Assert.True(DateTime.UtcNow < until, $"{folder}: {built}");
await Task.Delay(50);
}
}
/// <summary>Waits until no job is waiting or running.</summary>
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
private static async Task UntilScannedAsync(TestServer server)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var state in new[] { JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, "the scan at start did not finish");
await Task.Delay(50);
}
}
}
[Test]
public async Task ASequenceCreatedThroughTheApiTakesUploadsIntoAFolderOfItsOwn()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
var (_, _, token) = await server.UserAsync("alberto", "editor");
var source = context.CreateTempDirectory("source");
var photos = new List<(string Path, byte[] Bytes)>();
for (var i = 0; i < 2; i++)
{
var path = Path.Combine(source, $"IMG_000{i + 1}.JPG");
var bytes = Photo(i + 1);
await File.WriteAllBytesAsync(path, bytes);
photos.Add((path, bytes));
}
// The Uploader's "New sequence" target: create it, then upload into it (BRIEF §15).
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "Titano Uploader (tests)") { Token = token };
var created = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null), context.Cancellation);
var first = Assert.NotNull(await client.UploadFileAsync(photos[0].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 11), ct: context.Cancellation));
var second = Assert.NotNull(await client.UploadFileAsync(photos[1].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 21), ct: context.Cancellation));
Assert.Equal((created.Id, created.Id, 0, 1), (first.SequenceId, second.SequenceId, first.Index, second.Index));
var row = Assert.NotNull(await server.Database.Sequences.GetAsync(created.Id));
Assert.True(row.Folder.StartsWith("2026/2026-09-28", StringComparison.Ordinal), row.Folder);
Assert.Equal(("Harbour sunset", 2), (row.Title, row.FrameCount));
var sidecar = ManifestStore.ReadSequence(new ArchivePaths(server.Options.ArchivePath).Sequence(row.Folder).ManifestPath);
Assert.Equal((created.Id, "Harbour sunset"), (sidecar.Id, sidecar.Title));
Assert.SequenceEqual([first.FrameId, second.FrameId], sidecar.Frames.Select(f => f.Id));
Assert.FileMissing(Path.Combine(server.Options.ArchivePath, ".titano", "sequence.json"));
for (var i = 0; i < 2; i++)
{
using var downloaded = new MemoryStream();
await client.DownloadFrameAsync(i == 0 ? first.FrameId : second.FrameId, "original", downloaded, ct: context.Cancellation);
Assert.BytesEqual(photos[i].Bytes, downloaded.ToArray());
}
}
[Test]
public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var photo = Photo(3);
// Room for the photo once: a repair puts back bytes the archive already counts.
var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: photo.Length + 1000);
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
var path = Path.Combine(server.Options.ArchivePath, row.RelativePath);
var known = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof", expected: HttpStatusCode.OK);
Assert.Equal((true, frame.FrameId), (known.AlreadyExists, known.FrameId));
File.Delete(path);
// No "already exists" for a copy that is gone: the bytes come again and the frame gets its file back.
var again = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount);
using var me = await editor.GetAsync("/api/v1/auth/me");
Assert.Equal(photo.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes);
}
[Test]
public async Task AnIntactCopyElsewhereDoesNotStandForTheDamagedOneOfTheTargetSequence()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo(4);
var roof = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var path = Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(roof.FrameId)).RelativePath);
File.Delete(path);
// With the roof's copy gone, the same photo is filed into a second sequence.
var yard = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "yard");
Assert.NotEqual(roof.SequenceId, yard.SequenceId);
// Into the roof sequence the yard's copy is no shortcut, not even for a commit a crash interrupted: the bytes come
// and the roof's frame gets its file back.
var start = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", roof.SequenceId);
var id = Assert.NotNull(start.UploadId);
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
for (var i = 0; i * start.ChunkSize < photo.Length; i++)
{
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(photo, start.ChunkSize, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: context.Cancellation));
await server.Services.GetRequiredService<UploadMaintenance>().RecoverAsync(context.Cancellation);
using var complete = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
var again = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse);
Assert.Equal((roof.FrameId, roof.Index), (again.FrameId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
// Each sequence's own copy answers for it.
Assert.Equal((true, roof.FrameId), await ShortcutAsync(editor, photo, "roof"));
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
}
private static async Task<(bool AlreadyExists, Guid? FrameId)> ShortcutAsync(HttpClient client, byte[] photo, string sessionKey)
{
var start = await Uploads.StartAsync(client, photo, "IMG_0001.JPG", sessionKey: sessionKey, expected: HttpStatusCode.OK);
return (start.AlreadyExists, start.FrameId);
}
private static byte[] Photo(int seed = 0)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
}