Files
Chronos/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
T
Claude c68590920f fix(archive): the admin form reads archive.json; a poll never undoes an admin's edit
GET /api/v1/archive/settings refreshes the served archive from
archive.json first, so the admin form (which saves every field) starts
from a desktop's change made since the last poll instead of writing the
old templates back. ServerArchive.RefreshAsync replaces the served
archive only when the refresh registered a change, and only through a
compare-and-swap against the archive it read, so a poll racing an
admin's PUT no longer puts the pre-edit row back for ten minutes.

RefreshSettingsAsync now ignores an archive.json a newer Titano wrote,
as documented, instead of registering what this version reads of it.

AppServicesTests now points the folder at an archive the index does not
list first, and checks that the archive no longer chosen gets no scan.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 16:33:56 +00:00

281 lines
16 KiB
C#

using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Model;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
using Titano.Server.Storage;
using Titano.Testing;
using UploadState = Titano.Data.Repositories.UploadState;
namespace Titano.Server.Tests.Storage;
/// <summary>The production composition: uploads committed by the real archive engine, thumbnails built by its job.</summary>
[Category("server")]
public sealed class ArchiveModuleTests(TestContext context)
{
[Test]
public async Task UploadsLandInTheArchiveLayoutAndGetThumbnails()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo();
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
var sequence = Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId));
var paths = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder);
Assert.True(File.Exists(paths.ManifestPath), "sequence.json");
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, row.RelativePath)));
Assert.Count(0, Directory.GetFiles(Path.Combine(server.Options.ArchivePath, ".titano", "staging")));
// The commit queues a thumbnail job; its handler comes from the archive module.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (true)
{
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.FrameId}/thumb");
if (thumb.StatusCode == HttpStatusCode.OK)
{
var jpeg = await thumb.Content.ReadAsByteArrayAsync();
Assert.True(jpeg is [0xFF, 0xD8, ..], "a JPEG thumbnail");
break;
}
Assert.True(DateTime.UtcNow < until, $"thumbnail not built: {(int)thumb.StatusCode}");
await Task.Delay(100);
}
}
[Test]
public async Task AServerOnSqliteBacksUpItsIndexOnStart()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var backups = Path.Combine(server.Options.DataPath, "backups");
// No copy yet, so the first one is made at once (BRIEF §6.2), named after the index it copies.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (!Directory.Exists(backups) || Directory.GetFiles(backups, "*.db").Length == 0)
{
Assert.True(DateTime.UtcNow < until, "no backup of the index");
await Task.Delay(50);
}
var path = Assert.NotNull(((Titano.Data.Sqlite.SqliteDatabase)server.Database.Database).Path);
var backup = Titano.Data.DatabaseBackups.List(backups, path).Single();
var summary = await Titano.Data.DatabaseBackups.SummarizeBackupAsync(backup.Path, context.CreateTempDirectory("scratch"), context.Cancellation);
Assert.Equal(server.Database.SchemaVersion, summary.SchemaVersion);
}
[Test]
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling()
{
var photo = Photo();
// An adopted folder that already holds photos.
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3));
var polling = server.Services.GetRequiredService<ArchivePolling>();
var paused = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Scan,
TargetIds = [archive.Id],
State = JobState.Paused,
Created = DateTimeOffset.UtcNow,
};
await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation);
Assert.Null(await polling.PollAsync(context.Cancellation));
await server.Services.GetRequiredService<JobEngineHost>().Engine.CancelAsync(paused.Id, context.Cancellation);
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
Assert.Equal(JobPriority.Background, scan.Priority);
await UntilIndexedAsync(server, archive.Id, "garden");
}
[Test]
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var archive = server.Services.GetRequiredService<ServerArchive>();
var polling = server.Services.GetRequiredService<ArchivePolling>();
// Nothing changed: the served archive (the one an administrator's edit put there) is not replaced by a row read again.
var served = archive.Current;
await polling.PollAsync(context.Cancellation);
Assert.True(ReferenceEquals(served, archive.Current), "a poll that registered nothing keeps the served archive");
// A desktop sharing the archive changed its folder template and proxy size in archive.json.
var path = new ArchivePaths(archive.Root).ManifestPath;
var manifest = ManifestStore.ReadArchive(path);
ManifestStore.WriteArchive(path, manifest with
{
LayoutTemplates = manifest.LayoutTemplates with { Folder = "{camera}/{date}" },
ProxyPolicy = manifest.ProxyPolicy with { LongEdge = 2560 },
});
await polling.PollAsync(context.Cancellation);
// Uploads (the index row) and the admin panel (the served archive) follow it.
Assert.Equal(("{camera}/{date}", 2560), (archive.Current.FolderTemplate, archive.Current.ProxyLongEdge));
var row = Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, context.Cancellation));
Assert.Equal(("{camera}/{date}", 2560), (row.FolderTemplate, row.ProxyLongEdge));
// The admin panel opens before the next poll: its form, which saves every field, starts from the desktop's change.
ManifestStore.WriteArchive(path, ManifestStore.ReadArchive(path) with { LayoutTemplates = manifest.LayoutTemplates with { Folder = "{year}/{camera}" } });
var admin = await server.AdminAsync();
using (var read = await admin.GetAsync("/api/v1/archive/settings"))
Assert.Equal("{year}/{camera}", (await read.ReadAsync(ProtocolJson.Default.ArchiveSettingsDto)).FolderTemplate);
Assert.Equal("{year}/{camera}", archive.Current.FolderTemplate);
// A refresh that read the archive before an administrator saved an edit never puts it back over that edit.
var before = archive.Current;
archive.Current = before with { Name = "Edited" };
Assert.False(archive.TryReplace(before, before with { Name = "Stale" }), "the edit saved meanwhile stays");
Assert.Equal("Edited", archive.Current.Name);
}
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder))
{
Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed");
await Task.Delay(50);
}
}
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
private static async Task UntilScannedAsync(TestServer server)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var state in new[] { JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, "the scan at start did not finish");
await Task.Delay(50);
}
}
}
[Test]
public async Task ASequenceCreatedThroughTheApiTakesUploadsIntoAFolderOfItsOwn()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
var (_, _, token) = await server.UserAsync("alberto", "editor");
var source = context.CreateTempDirectory("source");
var photos = new List<(string Path, byte[] Bytes)>();
for (var i = 0; i < 2; i++)
{
var path = Path.Combine(source, $"IMG_000{i + 1}.JPG");
var bytes = Photo(i + 1);
await File.WriteAllBytesAsync(path, bytes);
photos.Add((path, bytes));
}
// The Uploader's "New sequence" target: create it, then upload into it (BRIEF §15).
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "Titano Uploader (tests)") { Token = token };
var created = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null), context.Cancellation);
var first = Assert.NotNull(await client.UploadFileAsync(photos[0].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 11), ct: context.Cancellation));
var second = Assert.NotNull(await client.UploadFileAsync(photos[1].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 21), ct: context.Cancellation));
Assert.Equal((created.Id, created.Id, 0, 1), (first.SequenceId, second.SequenceId, first.Index, second.Index));
var row = Assert.NotNull(await server.Database.Sequences.GetAsync(created.Id));
Assert.True(row.Folder.StartsWith("2026/2026-09-28", StringComparison.Ordinal), row.Folder);
Assert.Equal(("Harbour sunset", 2), (row.Title, row.FrameCount));
var sidecar = ManifestStore.ReadSequence(new ArchivePaths(server.Options.ArchivePath).Sequence(row.Folder).ManifestPath);
Assert.Equal((created.Id, "Harbour sunset"), (sidecar.Id, sidecar.Title));
Assert.SequenceEqual([first.FrameId, second.FrameId], sidecar.Frames.Select(f => f.Id));
Assert.FileMissing(Path.Combine(server.Options.ArchivePath, ".titano", "sequence.json"));
for (var i = 0; i < 2; i++)
{
using var downloaded = new MemoryStream();
await client.DownloadFrameAsync(i == 0 ? first.FrameId : second.FrameId, "original", downloaded, ct: context.Cancellation);
Assert.BytesEqual(photos[i].Bytes, downloaded.ToArray());
}
}
[Test]
public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var photo = Photo(3);
// Room for the photo once: a repair puts back bytes the archive already counts.
var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: photo.Length + 1000);
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
var path = Path.Combine(server.Options.ArchivePath, row.RelativePath);
var known = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof", expected: HttpStatusCode.OK);
Assert.Equal((true, frame.FrameId), (known.AlreadyExists, known.FrameId));
File.Delete(path);
// No "already exists" for a copy that is gone: the bytes come again and the frame gets its file back.
var again = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount);
using var me = await editor.GetAsync("/api/v1/auth/me");
Assert.Equal(photo.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes);
}
[Test]
public async Task AnIntactCopyElsewhereDoesNotStandForTheDamagedOneOfTheTargetSequence()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo(4);
var roof = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var path = Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(roof.FrameId)).RelativePath);
File.Delete(path);
// With the roof's copy gone, the same photo is filed into a second sequence.
var yard = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "yard");
Assert.NotEqual(roof.SequenceId, yard.SequenceId);
// Into the roof sequence the yard's copy is no shortcut, not even for a commit a crash interrupted: the bytes come
// and the roof's frame gets its file back.
var start = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", roof.SequenceId);
var id = Assert.NotNull(start.UploadId);
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
for (var i = 0; i * start.ChunkSize < photo.Length; i++)
{
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(photo, start.ChunkSize, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: context.Cancellation));
await server.Services.GetRequiredService<UploadMaintenance>().RecoverAsync(context.Cancellation);
using var complete = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
var again = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse);
Assert.Equal((roof.FrameId, roof.Index), (again.FrameId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
// Each sequence's own copy answers for it.
Assert.Equal((true, roof.FrameId), await ShortcutAsync(editor, photo, "roof"));
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
}
private static async Task<(bool AlreadyExists, Guid? FrameId)> ShortcutAsync(HttpClient client, byte[] photo, string sessionKey)
{
var start = await Uploads.StartAsync(client, photo, "IMG_0001.JPG", sessionKey: sessionKey, expected: HttpStatusCode.OK);
return (start.AlreadyExists, start.FrameId);
}
private static byte[] Photo(int seed = 0)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
}