- The desktop's rescans run for the whole session and each one targets the archive at Settings › Archive's folder (as the Import and Archive screens do), so an archive created on the first run, chosen in Settings, relinked or imported with settings is rescanned from the next tick on, and Rescan every always reaches a live timer. The folder of the start is reopened (relink, interrupted renames) as soon as its archive.json is there, so an unplugged drive or a folder without an archive no longer stops polling for the session. - ArchiveService.RefreshSettingsAsync reads archive.json and registers the archive again when its name, templates or proxy size changed elsewhere (a server's admin panel, another PC); null when the file cannot be read, is newer or belongs to another archive. Desktop rescans and the server's polling call it first, so imports, uploads and the admin panel follow the shared settings without a restart. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
262 lines
14 KiB
C#
262 lines
14 KiB
C#
using System.Net;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Titano.Core.Archiving;
|
|
using Titano.Core.Codecs.Jpeg;
|
|
using Titano.Core.Model;
|
|
using Titano.Protocol;
|
|
using Titano.Protocol.Client;
|
|
using Titano.Server.Jobs;
|
|
using Titano.Server.Storage;
|
|
using Titano.Testing;
|
|
using UploadState = Titano.Data.Repositories.UploadState;
|
|
|
|
namespace Titano.Server.Tests.Storage;
|
|
|
|
/// <summary>The production composition: uploads committed by the real archive engine, thumbnails built by its job.</summary>
|
|
[Category("server")]
|
|
public sealed class ArchiveModuleTests(TestContext context)
|
|
{
|
|
[Test]
|
|
public async Task UploadsLandInTheArchiveLayoutAndGetThumbnails()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var photo = Photo();
|
|
|
|
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
|
|
|
|
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
|
|
var sequence = Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId));
|
|
var paths = new ArchivePaths(server.Options.ArchivePath).Sequence(sequence.Folder);
|
|
Assert.True(File.Exists(paths.ManifestPath), "sequence.json");
|
|
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, row.RelativePath)));
|
|
Assert.Count(0, Directory.GetFiles(Path.Combine(server.Options.ArchivePath, ".titano", "staging")));
|
|
|
|
// The commit queues a thumbnail job; its handler comes from the archive module.
|
|
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
|
while (true)
|
|
{
|
|
using var thumb = await editor.GetAsync($"/api/v1/frames/{frame.FrameId}/thumb");
|
|
if (thumb.StatusCode == HttpStatusCode.OK)
|
|
{
|
|
var jpeg = await thumb.Content.ReadAsByteArrayAsync();
|
|
Assert.True(jpeg is [0xFF, 0xD8, ..], "a JPEG thumbnail");
|
|
break;
|
|
}
|
|
Assert.True(DateTime.UtcNow < until, $"thumbnail not built: {(int)thumb.StatusCode}");
|
|
await Task.Delay(100);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerOnSqliteBacksUpItsIndexOnStart()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var backups = Path.Combine(server.Options.DataPath, "backups");
|
|
|
|
// No copy yet, so the first one is made at once (BRIEF §6.2), named after the index it copies.
|
|
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
|
while (!Directory.Exists(backups) || Directory.GetFiles(backups, "*.db").Length == 0)
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, "no backup of the index");
|
|
await Task.Delay(50);
|
|
}
|
|
var path = Assert.NotNull(((Titano.Data.Sqlite.SqliteDatabase)server.Database.Database).Path);
|
|
var backup = Titano.Data.DatabaseBackups.List(backups, path).Single();
|
|
var summary = await Titano.Data.DatabaseBackups.SummarizeBackupAsync(backup.Path, context.CreateTempDirectory("scratch"), context.Cancellation);
|
|
Assert.Equal(server.Database.SchemaVersion, summary.SchemaVersion);
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosAlreadyInTheArchiveAndAddedLaterAreIndexedByPolling()
|
|
{
|
|
var photo = Photo();
|
|
// An adopted folder that already holds photos.
|
|
await using var server = await TestServer.StartAsync(context, options =>
|
|
{
|
|
Directory.CreateDirectory(Path.Combine(options.ArchivePath, "roof"));
|
|
File.WriteAllBytes(Path.Combine(options.ArchivePath, "roof", "IMG_0001.JPG"), photo);
|
|
}, ArchiveModule.Register);
|
|
var archive = server.Services.GetRequiredService<ServerArchive>();
|
|
await UntilIndexedAsync(server, archive.Id, "roof");
|
|
await UntilScannedAsync(server);
|
|
|
|
// Another writer adds a folder: the next poll queues a scan, but not while one is pending.
|
|
Directory.CreateDirectory(Path.Combine(server.Options.ArchivePath, "garden"));
|
|
File.WriteAllBytes(Path.Combine(server.Options.ArchivePath, "garden", "IMG_0002.JPG"), Photo(seed: 3));
|
|
var polling = server.Services.GetRequiredService<ArchivePolling>();
|
|
var paused = new Job
|
|
{
|
|
Id = Guid.CreateVersion7(),
|
|
Kind = JobKind.Scan,
|
|
TargetIds = [archive.Id],
|
|
State = JobState.Paused,
|
|
Created = DateTimeOffset.UtcNow,
|
|
};
|
|
await server.Database.Jobs.EnqueueAsync(paused, context.Cancellation);
|
|
Assert.Null(await polling.PollAsync(context.Cancellation));
|
|
await server.Services.GetRequiredService<JobEngineHost>().Engine.CancelAsync(paused.Id, context.Cancellation);
|
|
|
|
var scan = Assert.NotNull(await polling.PollAsync(context.Cancellation));
|
|
Assert.Equal(JobPriority.Background, scan.Priority);
|
|
await UntilIndexedAsync(server, archive.Id, "garden");
|
|
}
|
|
|
|
[Test]
|
|
public async Task SettingsADesktopChangedInArchiveJsonReachTheServerAtTheNextPoll()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var archive = server.Services.GetRequiredService<ServerArchive>();
|
|
// A desktop sharing the archive changed its folder template and proxy size in archive.json.
|
|
var path = new ArchivePaths(archive.Root).ManifestPath;
|
|
var manifest = ManifestStore.ReadArchive(path);
|
|
ManifestStore.WriteArchive(path, manifest with
|
|
{
|
|
LayoutTemplates = manifest.LayoutTemplates with { Folder = "{camera}/{date}" },
|
|
ProxyPolicy = manifest.ProxyPolicy with { LongEdge = 2560 },
|
|
});
|
|
|
|
await server.Services.GetRequiredService<ArchivePolling>().PollAsync(context.Cancellation);
|
|
|
|
// Uploads (the index row) and the admin panel (the served archive) follow it.
|
|
Assert.Equal(("{camera}/{date}", 2560), (archive.Current.FolderTemplate, archive.Current.ProxyLongEdge));
|
|
var row = Assert.NotNull(await server.Database.Archives.GetAsync(archive.Id, context.Cancellation));
|
|
Assert.Equal(("{camera}/{date}", 2560), (row.FolderTemplate, row.ProxyLongEdge));
|
|
}
|
|
|
|
private static async Task UntilIndexedAsync(TestServer server, Guid archiveId, string folder)
|
|
{
|
|
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
|
while (!(await server.Database.Sequences.ListLocationsAsync(archiveId)).Any(l => l.Folder == folder))
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, $"{folder} was not indexed");
|
|
await Task.Delay(50);
|
|
}
|
|
}
|
|
|
|
/// <summary>Waits for the scan queued at start to finish: its sequences are indexed before it ends.</summary>
|
|
private static async Task UntilScannedAsync(TestServer server)
|
|
{
|
|
var until = DateTime.UtcNow + TimeSpan.FromSeconds(30);
|
|
foreach (var state in new[] { JobState.Queued, JobState.Running })
|
|
{
|
|
while ((await server.Database.Jobs.ListAsync(state, JobKind.Scan)).Items.Count > 0)
|
|
{
|
|
Assert.True(DateTime.UtcNow < until, "the scan at start did not finish");
|
|
await Task.Delay(50);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task ASequenceCreatedThroughTheApiTakesUploadsIntoAFolderOfItsOwn()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
|
|
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
|
var source = context.CreateTempDirectory("source");
|
|
var photos = new List<(string Path, byte[] Bytes)>();
|
|
for (var i = 0; i < 2; i++)
|
|
{
|
|
var path = Path.Combine(source, $"IMG_000{i + 1}.JPG");
|
|
var bytes = Photo(i + 1);
|
|
await File.WriteAllBytesAsync(path, bytes);
|
|
photos.Add((path, bytes));
|
|
}
|
|
|
|
// The Uploader's "New sequence" target: create it, then upload into it (BRIEF §15).
|
|
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "Titano Uploader (tests)") { Token = token };
|
|
var created = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null), context.Cancellation);
|
|
var first = Assert.NotNull(await client.UploadFileAsync(photos[0].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 11), ct: context.Cancellation));
|
|
var second = Assert.NotNull(await client.UploadFileAsync(photos[1].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 21), ct: context.Cancellation));
|
|
|
|
Assert.Equal((created.Id, created.Id, 0, 1), (first.SequenceId, second.SequenceId, first.Index, second.Index));
|
|
var row = Assert.NotNull(await server.Database.Sequences.GetAsync(created.Id));
|
|
Assert.True(row.Folder.StartsWith("2026/2026-09-28", StringComparison.Ordinal), row.Folder);
|
|
Assert.Equal(("Harbour sunset", 2), (row.Title, row.FrameCount));
|
|
var sidecar = ManifestStore.ReadSequence(new ArchivePaths(server.Options.ArchivePath).Sequence(row.Folder).ManifestPath);
|
|
Assert.Equal((created.Id, "Harbour sunset"), (sidecar.Id, sidecar.Title));
|
|
Assert.SequenceEqual([first.FrameId, second.FrameId], sidecar.Frames.Select(f => f.Id));
|
|
Assert.FileMissing(Path.Combine(server.Options.ArchivePath, ".titano", "sequence.json"));
|
|
for (var i = 0; i < 2; i++)
|
|
{
|
|
using var downloaded = new MemoryStream();
|
|
await client.DownloadFrameAsync(i == 0 ? first.FrameId : second.FrameId, "original", downloaded, ct: context.Cancellation);
|
|
Assert.BytesEqual(photos[i].Bytes, downloaded.ToArray());
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var photo = Photo(3);
|
|
// Room for the photo once: a repair puts back bytes the archive already counts.
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: photo.Length + 1000);
|
|
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
|
|
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
|
|
var path = Path.Combine(server.Options.ArchivePath, row.RelativePath);
|
|
var known = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof", expected: HttpStatusCode.OK);
|
|
Assert.Equal((true, frame.FrameId), (known.AlreadyExists, known.FrameId));
|
|
|
|
File.Delete(path);
|
|
// No "already exists" for a copy that is gone: the bytes come again and the frame gets its file back.
|
|
var again = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
|
|
|
|
Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index));
|
|
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
|
|
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount);
|
|
using var me = await editor.GetAsync("/api/v1/auth/me");
|
|
Assert.Equal(photo.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnIntactCopyElsewhereDoesNotStandForTheDamagedOneOfTheTargetSequence()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
|
var photo = Photo(4);
|
|
var roof = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
|
|
var path = Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(roof.FrameId)).RelativePath);
|
|
File.Delete(path);
|
|
// With the roof's copy gone, the same photo is filed into a second sequence.
|
|
var yard = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "yard");
|
|
Assert.NotEqual(roof.SequenceId, yard.SequenceId);
|
|
|
|
// Into the roof sequence the yard's copy is no shortcut, not even for a commit a crash interrupted: the bytes come
|
|
// and the roof's frame gets its file back.
|
|
var start = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", roof.SequenceId);
|
|
var id = Assert.NotNull(start.UploadId);
|
|
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
|
|
for (var i = 0; i * start.ChunkSize < photo.Length; i++)
|
|
{
|
|
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(photo, start.ChunkSize, i));
|
|
await put.ExpectAsync(HttpStatusCode.OK);
|
|
}
|
|
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: context.Cancellation));
|
|
await server.Services.GetRequiredService<UploadMaintenance>().RecoverAsync(context.Cancellation);
|
|
using var complete = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null);
|
|
await complete.ExpectAsync(HttpStatusCode.OK);
|
|
var again = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse);
|
|
|
|
Assert.Equal((roof.FrameId, roof.Index), (again.FrameId, again.Index));
|
|
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
|
|
// Each sequence's own copy answers for it.
|
|
Assert.Equal((true, roof.FrameId), await ShortcutAsync(editor, photo, "roof"));
|
|
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
|
|
}
|
|
|
|
private static async Task<(bool AlreadyExists, Guid? FrameId)> ShortcutAsync(HttpClient client, byte[] photo, string sessionKey)
|
|
{
|
|
var start = await Uploads.StartAsync(client, photo, "IMG_0001.JPG", sessionKey: sessionKey, expected: HttpStatusCode.OK);
|
|
return (start.AlreadyExists, start.FrameId);
|
|
}
|
|
|
|
private static byte[] Photo(int seed = 0)
|
|
{
|
|
const int width = 96, height = 64;
|
|
var pixels = new byte[width * height * 3];
|
|
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
|
|
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
|
|
}
|
|
}
|