using System.Net;
using System.Security.Cryptography;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Archiving;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Jobs;
using Titano.Server.Storage;
using Titano.Testing;
using UploadState = Titano.Data.Repositories.UploadState;
namespace Titano.Server.Tests.Storage;
/// The production composition: uploads committed by the real archive engine, thumbnails built by its job.
[Category("server")]
public sealed class ArchiveModuleTests(TestContext context)
{
[Test]
public async Task UploadsLandInTheArchiveLayoutAndGetThumbnails()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo();
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
var sequence = Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId));
var paths = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder);
Assert.True(File.Exists(paths.ManifestPath), "sequence.json");
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, row.RelativePath)));
Assert.Count(0, Directory.GetFiles(Path.Combine(server.Options.ArchivePath, ".titano", "staging")));
// The commit queues a thumbnail job; its handler comes from the archive module.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (true)
{
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.FrameId}/thumb");
if (thumb.StatusCode == HttpStatusCode.OK)
{
var jpeg = await thumb.Content.ReadAsByteArrayAsync();
Assert.True(jpeg is [0xFF, 0xD8, ..], "a JPEG thumbnail");
break;
}
Assert.True(DateTime.UtcNow < until, $"thumbnail not built: {(int)thumb.StatusCode}");
await Task.Delay(100);
}
}
[Test]
public async Task AServerOnSqliteBacksUpItsIndexOnStart()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var backups = Path.Combine(server.Options.DataPath, "backups");
// No copy yet, so the first one is made at once (BRIEF §6.2), named after the index it copies.
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (!Directory.Exists(backups) || Directory.GetFiles(backups, "*.db").Length == 0)
{
Assert.True(DateTime.UtcNow < until, "no backup of the index");
await Task.Delay(50);
}
var path = Assert.NotNull(((Titano.Data.Sqlite.SqliteDatabase)server.Database.Database).Path);
var backup = Titano.Data.DatabaseBackups.List(backups, path).Single();
var summary = await Titano.Data.DatabaseBackups.SummarizeBackupAsync(backup.Path, context.CreateTempDirectory("scratch"), context.Cancellation);
Assert.Equal(server.Database.SchemaVersion, summary.SchemaVersion);
}
[Test]
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPollingWithThumbnailsAndProxies()
{
var photo = Photo();
// An adopted folder that already holds photos.
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
// The scan queues the thumbnails and proxies clients load from the server, as an upload does.
await UntilImagesAsync(server, editor, archive.Id, "roof");
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3));
var polling = server.Services.GetRequiredService();
var paused = new Job
{
Id = Guid.CreateVersion7(),
Kind = JobKind.Scan,
TargetIds = [archive.Id],
State = JobState.Paused,
Created = DateTimeOffset.UtcNow,
};
await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation);
Assert.Null(await polling.PollAsync(context.Cancellation));
await server.Services.GetRequiredService().Engine.CancelAsync(paused.Id, context.Cancellation);
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
Assert.Equal(JobPriority.Background, scan.Priority);
await UntilIndexedAsync(server, archive.Id, "garden");
await UntilImagesAsync(server, editor, archive.Id, "garden");
}
[Test]
public async Task FramesADesktopSharingTheArchiveAddsToASequenceGetThumbnailsAndProxiesAfterThePoll()
{
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0002.JPG"), Photo());
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
await UntilImagesAsync(server, editor, archive.Id, "roof");
await UntilIdleAsync(server);
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
// A desktop sharing the archive, with an index of its own, imports a photo taken earlier into the sequence: it files
// the photo and lists it in the sidecar itself, keeping the sidecar's ThumbsBuilt and ProxiesBuilt.
var desktopFolder = context.CreateTempDirectory("desktop");
await using (var desktopIndex = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(desktopFolder, "titano.db")), context.Cancellation))
{
var desktop = new ArchiveService(desktopIndex, new ArchiveServiceOptions { Owner = "desktop" });
var shared = await desktop.OpenAsync(server.Options.ArchivePath, context.Cancellation);
await desktop.Indexer.ScanAsync(shared, ScanMode.Incremental, null, _ => ValueTask.CompletedTask, context.Cancellation);
var photo = Photo(seed: 5);
var staged = Path.Combine(desktopFolder, "IMG_0001.JPG");
await File.WriteAllBytesAsync(staged, photo);
await desktop.Ingest.CommitAsync(new IngestRequest(shared.Id, roof.Id, null, null, staged, "IMG_0001.JPG", photo.Length,
Convert.ToHexStringLower(SHA256.HashData(photo)), new DateTime(2026, 9, 28, 18, 0, 0), null), context.Cancellation);
}
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
// The next poll indexes it and queues the images: the new frame and the one it moved get theirs.
Assert.NotNull(await server.Services.GetRequiredService().PollAsync(context.Cancellation));
await UntilScannedAsync(server);
Assert.Equal(2, Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).FrameCount);
await UntilImagesAsync(server, editor, archive.Id, "roof");
}
[Test]
public async Task AThumbnailDrawnBeforeAVerifyHashedThePhotoIsStillServed()
{
await using var server = await TestServer.StartAsync(context, options =>
{
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), Photo());
}, ArchiveModule.Register);
var archive = server.Services.GetRequiredService();
var (editor, _, _) = await server.UserAsync("alberto", "editor");
await UntilIndexedAsync(server, archive.Id, "roof");
await UntilScannedAsync(server);
await UntilImagesAsync(server, editor, archive.Id, "roof");
await UntilIdleAsync(server);
var roof = (await server.Database.Sequences.ListLocationsAsync(archive.Id)).Single(l => l.Folder == "roof");
var frame = (await server.Database.Frames.ListBySequenceAsync(roof.Id)).Single();
Assert.Null(frame.Sha256);
// A verify gives the photo the scan adopted its first hash; the thumbnail drawn from it before is still its own.
var verified = await server.Services.GetRequiredService().Verifier.VerifyAsync(
Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id)), [roof.Id], default, (_, _, _) => ValueTask.CompletedTask, null,
context.Cancellation);
Assert.Equal(1L, verified.Hashed);
Assert.NotNull(Assert.NotNull(await server.Database.Frames.GetAsync(frame.Id)).Sha256);
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.Id}/thumb");
Assert.Equal(HttpStatusCode.OK, thumb.StatusCode);
Assert.True(Assert.NotNull(await server.Database.Sequences.GetAsync(roof.Id)).Status.HasFlag(SequenceStatus.ThumbsBuilt), "built");
}
[Test]
public async Task SettingsADesktopChangedInArchiveJsonReachUploadsAndTheAdminPanel()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var archive = server.Services.GetRequiredService();
var polling = server.Services.GetRequiredService();
// Nothing changed: the served archive (the one an administrator's edit put there) is not replaced by a row read again.
var served = archive.Current;
await polling.PollAsync(context.Cancellation);
Assert.True(ReferenceEquals(served, archive.Current), "a poll that registered nothing keeps the served archive");
// A desktop sharing the archive changed its folder template and proxy size in archive.json.
var path = new ArchivePaths(archive.Root).ManifestPath;
var manifest = ManifestStore.ReadArchive(path);
ManifestStore.WriteArchive(path, manifest with
{
LayoutTemplates = manifest.LayoutTemplates with { Folder = "{camera}/{date}" },
ProxyPolicy = manifest.ProxyPolicy with { LongEdge = 2560 },
});
await polling.PollAsync(context.Cancellation);
// Uploads (the index row) and the admin panel (the served archive) follow it.
Assert.Equal(("{camera}/{date}", 2560), (archive.Current.FolderTemplate, archive.Current.ProxyLongEdge));
var row = Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, context.Cancellation));
Assert.Equal(("{camera}/{date}", 2560), (row.FolderTemplate, row.ProxyLongEdge));
// The admin panel opens before the next poll: its form, which saves every field, starts from the desktop's change.
ManifestStore.WriteArchive(path, ManifestStore.ReadArchive(path) with { LayoutTemplates = manifest.LayoutTemplates with { Folder = "{year}/{camera}" } });
var admin = await server.AdminAsync();
using (var read = await admin.GetAsync("/api/v1/archive/settings"))
Assert.Equal("{year}/{camera}", (await read.ReadAsync(ProtocolJson.Default.ArchiveSettingsDto)).FolderTemplate);
Assert.Equal("{year}/{camera}", archive.Current.FolderTemplate);
// A refresh that read the archive before an administrator saved an edit never puts it back over that edit.
var before = archive.Current;
archive.Current = before with { Name = "Edited" };
Assert.False(archive.TryReplace(before, before with { Name = "Stale" }), "the edit saved meanwhile stays");
Assert.Equal("Edited", archive.Current.Name);
}
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder))
{
Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed");
await Task.Delay(50);
}
}
///
/// Waits until every frame of the sequence in has its thumbnail and proxy served and the
/// passes that made them have marked the sequence (a pass writes its files before it marks).
///
private static async Task UntilImagesAsync(TestServer server, HttpClient client, Guid archiveId, string folder)
{
var sequence = (await server.Database.Sequences.ListLocationsAsync(archiveId)).Single(l => l.Folder == folder);
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var frame in await server.Database.Frames.ListBySequenceAsync(sequence.Id))
{
foreach (var asset in new[] { "thumb", "proxy" })
{
while (true)
{
using var response = await client.GetAsync($"/api/v1/frames/{frame.Id}/{asset}");
if (response.StatusCode == HttpStatusCode.OK)
{
Assert.True(await response.Content.ReadAsByteArrayAsync() is [0xFF, 0xD8, ..], $"a JPEG {asset} of {frame.FileName}");
break;
}
Assert.True(DateTime.UtcNow < until, $"no {asset} of {frame.FileName} in {folder}: {(int)response.StatusCode}");
await Task.Delay(100);
}
}
}
while (Assert.NotNull(await server.Database.Sequences.GetAsync(sequence.Id)).Status is var built
&& !(built.HasFlag(SequenceStatus.ThumbsBuilt) && built.HasFlag(SequenceStatus.ProxiesBuilt)))
{
Assert.True(DateTime.UtcNow < until, $"{folder}: {built}");
await Task.Delay(50);
}
}
/// Waits until no job is waiting or running.
private static async Task UntilIdleAsync(TestServer server)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var state in new[] { JobState.Queued, JobState.Running, JobState.Queued })
{
while ((await server.Database.Jobs.ListAsync(state)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, $"jobs still {state}");
await Task.Delay(50);
}
}
}
/// Waits for the scan queued at start to finish: its sequences are indexed before it ends.
private static async Task UntilScannedAsync(TestServer server)
{
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
foreach (var state in new[] { JobState.Queued, JobState.Running })
{
while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0)
{
Assert.True(DateTime.UtcNow < until, "the scan at start did not finish");
await Task.Delay(50);
}
}
}
[Test]
public async Task ASequenceCreatedThroughTheApiTakesUploadsIntoAFolderOfItsOwn()
{
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
var (_, _, token) = await server.UserAsync("alberto", "editor");
var source = context.CreateTempDirectory("source");
var photos = new List<(string Path, byte[] Bytes)>();
for (var i = 0; i < 2; i++)
{
var path = Path.Combine(source, $"IMG_000{i + 1}.JPG");
var bytes = Photo(i + 1);
await File.WriteAllBytesAsync(path, bytes);
photos.Add((path, bytes));
}
// The Uploader's "New sequence" target: create it, then upload into it (BRIEF §15).
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "Titano Uploader (tests)") { Token = token };
var created = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null), context.Cancellation);
var first = Assert.NotNull(await client.UploadFileAsync(photos[0].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 11), ct: context.Cancellation));
var second = Assert.NotNull(await client.UploadFileAsync(photos[1].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 21), ct: context.Cancellation));
Assert.Equal((created.Id, created.Id, 0, 1), (first.SequenceId, second.SequenceId, first.Index, second.Index));
var row = Assert.NotNull(await server.Database.Sequences.GetAsync(created.Id));
Assert.True(row.Folder.StartsWith("2026/2026-09-28", StringComparison.Ordinal), row.Folder);
Assert.Equal(("Harbour sunset", 2), (row.Title, row.FrameCount));
var sidecar = ManifestStore.ReadSequence(new ArchivePaths(server.Options.ArchivePath).Sequence(row.Folder).ManifestPath);
Assert.Equal((created.Id, "Harbour sunset"), (sidecar.Id, sidecar.Title));
Assert.SequenceEqual([first.FrameId, second.FrameId], sidecar.Frames.Select(f => f.Id));
Assert.FileMissing(Path.Combine(server.Options.ArchivePath, ".titano", "sequence.json"));
for (var i = 0; i < 2; i++)
{
using var downloaded = new MemoryStream();
await client.DownloadFrameAsync(i == 0 ? first.FrameId : second.FrameId, "original", downloaded, ct: context.Cancellation);
Assert.BytesEqual(photos[i].Bytes, downloaded.ToArray());
}
}
[Test]
public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var photo = Photo(3);
// Room for the photo once: a repair puts back bytes the archive already counts.
var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: photo.Length + 1000);
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
var path = Path.Combine(server.Options.ArchivePath, row.RelativePath);
var known = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof", expected: HttpStatusCode.OK);
Assert.Equal((true, frame.FrameId), (known.AlreadyExists, known.FrameId));
File.Delete(path);
// No "already exists" for a copy that is gone: the bytes come again and the frame gets its file back.
var again = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount);
using var me = await editor.GetAsync("/api/v1/auth/me");
Assert.Equal(photo.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes);
}
[Test]
public async Task AnIntactCopyElsewhereDoesNotStandForTheDamagedOneOfTheTargetSequence()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo(4);
var roof = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var path = Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(roof.FrameId)).RelativePath);
File.Delete(path);
// With the roof's copy gone, the same photo is filed into a second sequence.
var yard = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "yard");
Assert.NotEqual(roof.SequenceId, yard.SequenceId);
// Into the roof sequence the yard's copy is no shortcut, not even for a commit a crash interrupted: the bytes come
// and the roof's frame gets its file back.
var start = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", roof.SequenceId);
var id = Assert.NotNull(start.UploadId);
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
for (var i = 0; i * start.ChunkSize < photo.Length; i++)
{
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(photo, start.ChunkSize, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: context.Cancellation));
await server.Services.GetRequiredService().RecoverAsync(context.Cancellation);
using var complete = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
var again = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse);
Assert.Equal((roof.FrameId, roof.Index), (again.FrameId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
// Each sequence's own copy answers for it.
Assert.Equal((true, roof.FrameId), await ShortcutAsync(editor, photo, "roof"));
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
}
private static async Task<(bool AlreadyExists, Guid? FrameId)> ShortcutAsync(HttpClient client, byte[] photo, string sessionKey)
{
var start = await Uploads.StartAsync(client, photo, "IMG_0001.JPG", sessionKey: sessionKey, expected: HttpStatusCode.OK);
return (start.AlreadyExists, start.FrameId);
}
private static byte[] Photo(int seed = 0)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
}