diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index ad100883..750e260a 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -136,11 +136,12 @@ public sealed class ArchiveIngest : IArchiveIngest /// /// A sequence created without photos (POST /api/v1/sequences, BRIEF §14.3) has no folder until its first frame. - /// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence. + /// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence; one + /// that holds frames is never moved elsewhere, even while its root sidecar is missing. /// internal static bool AwaitsFolder(Sequence sequence, ArchivePaths paths) { - if (sequence.Folder.Length > 0) return false; + if (sequence.Folder.Length > 0 || sequence.FrameCount > 0) return false; var root = paths.Sequence(string.Empty); if (!File.Exists(root.ManifestPath)) return true; try @@ -363,7 +364,7 @@ public sealed class ArchiveIngest : IArchiveIngest foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); foreach (var (p, frame) in repairs.Where(r => repaired.Contains(r.Frame.Id))) - p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); + p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false) with { Repaired = true }); var unrepaired = repairs.Select(r => r.Frame.Id).Where(id => !repaired.Contains(id)).ToHashSet(); foreach (var (p, frameId) in duplicates) { diff --git a/src/Titano.Data/Archiving/IArchiveIngest.cs b/src/Titano.Data/Archiving/IArchiveIngest.cs index fa76d81d..b21a4b0a 100644 --- a/src/Titano.Data/Archiving/IArchiveIngest.cs +++ b/src/Titano.Data/Archiving/IArchiveIngest.cs @@ -19,9 +19,10 @@ public sealed record IngestRequest( /// /// The committed frame; when an intact frame of the sequence already held that SHA-256 (the file -/// stays in staging). A frame with that SHA-256 that was missing or damaged is repaired with the file instead (not a duplicate). +/// stays in staging). A frame with that SHA-256 that was missing or damaged is repaired with the file instead: +/// , not a duplicate, and the archive holds no more bytes than before. /// -public sealed record IngestResult(Guid SequenceId, Guid FrameId, int Index, string RelativePath, bool Duplicate); +public sealed record IngestResult(Guid SequenceId, Guid FrameId, int Index, string RelativePath, bool Duplicate, bool Repaired = false); /// /// Commits staged files into the archive layout (originals folder, naming templates, sequence.json written diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index d89aa320..f732b18d 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -65,18 +65,10 @@ public sealed class FrameRepository _db.ListAsync($"SELECT {Columns} FROM frames WHERE sequence_id = $1 AND idx > $2 ORDER BY idx LIMIT $3", [sequenceId, afterIndex, Cursor.ClampLimit(limit)], Map, cancellationToken); - /// Every frame with this content hash (duplicate detection). + /// Every frame with this content hash (duplicate detection, upload short-circuit). public Task> FindBySha256Async(string sha256, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant()], Map, cancellationToken); - /// - /// Frames with this content hash that are not flagged missing or corrupt: only such a copy may stand for an upload - /// (the upload short-circuit, a commit settled after a crash), so an upload of the same content repairs a damaged one. - /// - public Task> FindIntactBySha256Async(string sha256, CancellationToken cancellationToken = default) => - _db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 AND (flags & $2) = 0 ORDER BY sequence_id, idx", - [sha256.ToLowerInvariant(), (int)(FrameFlags.Missing | FrameFlags.Corrupt)], Map, cancellationToken); - /// Id and index of every frame of a sequence, in index order. public Task> ListPositionsAsync(Guid sequenceId, CancellationToken cancellationToken = default) => _db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)), diff --git a/src/Titano.Server/Storage/UploadMaintenance.cs b/src/Titano.Server/Storage/UploadMaintenance.cs index 2e53f72c..eedb4d82 100644 --- a/src/Titano.Server/Storage/UploadMaintenance.cs +++ b/src/Titano.Server/Storage/UploadMaintenance.cs @@ -1,3 +1,4 @@ +using Titano.Core.Model; using Titano.Data; using Titano.Data.Repositories; using Titano.Server.Configuration; @@ -45,16 +46,15 @@ public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, } /// - /// Uploads left committing by a crash: committed when an intact frame with their hash exists, otherwise back to - /// receiving (the partial file is intact, so the client simply completes again and the ingest files or repairs it). + /// Uploads left committing by a crash: committed when an intact frame stands for their content as at the start of an + /// upload (), otherwise back to receiving (the partial file is intact, so the client + /// simply completes again and the ingest files or repairs it). /// public async Task RecoverAsync(CancellationToken cancellationToken) { foreach (var upload in await db.Uploads.ListByStateAsync(UploadState.Committing, cancellationToken)) { - var frame = upload.JobId is null - ? (await db.Frames.FindIntactBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault(f => UploadRules.IsOnDisk(archive.Root, f)) - : null; + var frame = upload.JobId is null ? await StandInAsync(upload, cancellationToken) : null; if (frame is not null) { await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, frame.Id, cancellationToken); @@ -68,6 +68,15 @@ public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, } } + /// The intact frame that settles an interrupted commit, or null when the client must complete it again. + private async Task StandInAsync(Upload upload, CancellationToken cancellationToken) + { + var copies = await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken); + if (copies.Count == 0) return null; + var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken); + return UploadRules.StandIn(archive.Root, copies, target); + } + public async Task SweepAsync(CancellationToken cancellationToken) { var now = time.GetUtcNow(); diff --git a/src/Titano.Server/Storage/UploadRules.cs b/src/Titano.Server/Storage/UploadRules.cs index db3fccd9..a25d82fa 100644 --- a/src/Titano.Server/Storage/UploadRules.cs +++ b/src/Titano.Server/Storage/UploadRules.cs @@ -1,4 +1,5 @@ using Titano.Core.Model; +using Titano.Data; namespace Titano.Server.Storage; @@ -32,14 +33,40 @@ public static class UploadRules return received; } - /// - /// The frame's original is in the archive with its recorded size. Together with flags that say neither missing nor - /// corrupt (FindIntactBySha256Async), that is the copy an upload of the same content may stand for; a damaged - /// one is repaired by uploading the content again. - /// + /// The frame's original is in the archive with its recorded size. public static bool IsOnDisk(string archiveRoot, Frame frame) => ArchivePathGuard.ExistingFile(archiveRoot, frame.RelativePath) is { } file && file.Length == frame.Size; + /// + /// Not flagged missing or corrupt, and : the copy holds its content, so it may stand for an upload + /// of the same content; a damaged one is repaired by uploading the content again. + /// + public static bool IsIntact(string archiveRoot, Frame frame) => + (frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0 && IsOnDisk(archiveRoot, frame); + + /// + /// The frame that stands for an upload of content the archive already has (): an intact copy, + /// the one in the upload's sequence first. Null when there is none, or when the target holds + /// the content only in damaged copies: the upload's bytes repair them, whatever other sequences hold. + /// + public static Frame? StandIn(string archiveRoot, IReadOnlyList copies, Guid? target) + { + var intact = copies.Where(f => IsIntact(archiveRoot, f)).ToList(); + if (copies.Any(f => f.SequenceId == target) && !intact.Any(f => f.SequenceId == target)) return null; + return intact.FirstOrDefault(f => f.SequenceId == target) ?? intact.FirstOrDefault(); + } + + /// + /// The sequence an upload files into: its own, or the one its session key names (the user's own key without one); + /// null while that sequence does not exist yet. + /// + public static async Task TargetAsync(TitanoDatabase db, Guid archiveId, Guid? sequenceId, string? sessionKey, Guid userId, + CancellationToken cancellationToken) => + sequenceId ?? await db.Sequences.FindBySessionKeyAsync(archiveId, SessionKeyOf(sessionKey, userId), cancellationToken).ConfigureAwait(false); + + /// The session key an upload without a sequence is grouped by: the client's, or one per user. + public static string SessionKeyOf(string? sessionKey, Guid userId) => sessionKey ?? "user-" + userId.ToString("N"); + /// Free bytes on the volume holding , or null when the platform cannot tell. public static long? FreeBytes(string path) { diff --git a/src/Titano.Server/Storage/UploadService.cs b/src/Titano.Server/Storage/UploadService.cs index ebc57e30..dea51b7b 100644 --- a/src/Titano.Server/Storage/UploadService.cs +++ b/src/Titano.Server/Storage/UploadService.cs @@ -19,7 +19,7 @@ namespace Titano.Server.Storage; /// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an /// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the /// file on the archive volume, verifies its hash and hands it to , which repairs a missing or -/// damaged frame of that content. +/// damaged frame of that content (no quota is charged for a repair: the archive already counts those bytes). /// public sealed class UploadService( TitanoDatabase db, @@ -45,23 +45,23 @@ public sealed class UploadService( if (request.Size <= 0) return Fail(400, "bad_request", "size"); if (request.Size > options.MaxUploadBytes) return Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture)); if (request.SessionKey is { Length: > 200 }) return Fail(400, "bad_request", "sessionKey"); - if (request.SequenceId is { } target && await db.Sequences.GetAsync(target, cancellationToken) is null) + if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null) return Fail(404, "not_found", "sequence"); var sha = request.Sha256.ToLowerInvariant(); + var sessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim(); + var copies = await db.Frames.FindBySha256Async(sha, cancellationToken); + var target = copies.Count == 0 ? null : await UploadRules.TargetAsync(db, archive.Id, request.SequenceId, sessionKey, caller.Id, cancellationToken); // Only a copy that is still whole stands for the upload: a client may delete its file on this answer. - var known = (await db.Frames.FindIntactBySha256Async(sha, cancellationToken)).Where(f => UploadRules.IsOnDisk(archive.Root, f)).ToList(); - if (known.Count > 0) - { - var frame = known.FirstOrDefault(f => f.SequenceId == request.SequenceId) ?? known[0]; - return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, frame.Id); - } + if (UploadRules.StandIn(archive.Root, copies, target) is { } known) + return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id); var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken); if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id)) return Started(resumable); - if (caller.User.QuotaBytes > 0) + // A repair puts back bytes the archive already counts: no quota for it. + if (caller.User.QuotaBytes > 0 && !copies.Any(f => f.SequenceId == target)) { var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken); if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes) @@ -84,7 +84,7 @@ public sealed class UploadService( State = UploadState.Receiving, Created = now, UpdatedAt = now, - SessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim(), + SessionKey = sessionKey, }, cancellationToken); store.Create(upload.Id, upload.Size); return Started(upload); @@ -159,7 +159,7 @@ public sealed class UploadService( return Fail(409, "upload_state"); var user = await db.Users.GetAsync(caller.Id, stopping); - if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes) + if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes && !await HeldByTargetAsync(upload, stopping)) { await FailAsync(upload); return Fail(403, "quota_exceeded", user.QuotaBytes.ToString(CultureInfo.InvariantCulture)); @@ -175,7 +175,7 @@ public sealed class UploadService( archive.Id, upload.SequenceId, NewSequenceTitle: null, - upload.SequenceId is null ? upload.SessionKey ?? "user-" + caller.Id.ToString("N") : null, + upload.SequenceId is null ? UploadRules.SessionKeyOf(upload.SessionKey, caller.Id) : null, staged, upload.FileName, upload.Size, @@ -195,7 +195,7 @@ public sealed class UploadService( await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Committed, result.FrameId, CancellationToken.None); store.Delete(id); if (File.Exists(staged)) UploadStore.TryDelete(staged); - if (!result.Duplicate) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None); + if (!result.Duplicate && !result.Repaired) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None); if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } sequence) { await sequenceJobs.FramesAddedAsync(sequence, CancellationToken.None); @@ -235,6 +235,18 @@ public sealed class UploadService( store.Delete(upload.Id); } + /// + /// The sequence the upload files into holds its content already: filing it adds no bytes (an intact copy makes it a + /// duplicate, a damaged one is repaired with it). + /// + private async Task HeldByTargetAsync(Upload upload, CancellationToken cancellationToken) + { + var copies = await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken); + if (copies.Count == 0) return false; + var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken); + return copies.Any(f => f.SequenceId == target); + } + private StartUploadResponse Started(Upload upload) => new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null); diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index 2913ad7e..a944c0d9 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -143,6 +143,7 @@ public sealed class ArchiveIngestTests(TestContext context) var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct); Assert.False(repaired.Duplicate, "the server holds no intact copy: this one is filed"); + Assert.Equal((false, true), (original.Repaired, repaired.Repaired)); Assert.Equal((original.FrameId, original.Index, original.RelativePath), (repaired.FrameId, repaired.Index, repaired.RelativePath)); Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct)); Assert.FileMissing(again.Path); @@ -172,7 +173,7 @@ public sealed class ArchiveIngestTests(TestContext context) var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct); - Assert.False(repaired.Duplicate); + Assert.Equal((false, true), (repaired.Duplicate, repaired.Repaired)); Assert.Equal(original.FrameId, repaired.FrameId); Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct)); var aside = Directory.GetFiles(Path.GetDirectoryName(file)!, Path.GetFileName(file) + ".damaged-*"); @@ -220,6 +221,27 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.Equal(0, (await fx.ScanAsync()).Indexed); } + [Test] + public async Task ASequenceAdoptedAtTheRootIsNeverMovedWhenItsSidecarIsGone() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(fx.Archive.RootPath, Start, 2); + await fx.ScanAsync(); + var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault()); + Assert.Equal(string.Empty, adopted.Folder); + File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath); + + // It held photos, so it is no sequence waiting for a folder: the commit reports the missing sidecar. + var staged = fx.Stage(Start.AddMinutes(5), 3); + await Assert.ThrowsAsync(() => fx.Service.Ingest.CommitAsync(Request(fx, staged, sequenceId: adopted.Id), fx.Ct)); + + var row = (await fx.Db.Sequences.GetAsync(adopted.Id))!; + Assert.Equal((string.Empty, 2), (row.Folder, row.FrameCount)); + Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(adopted.Id)); + Assert.False(Directory.Exists(Path.Combine(fx.Archive.RootPath, "2026")), "no folder was claimed for it"); + Assert.FileExists(staged.Path); + } + [Test] public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong() { diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 0d58714b..81e7b40c 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,10 +1,12 @@ using System.Net; +using Microsoft.Extensions.DependencyInjection; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; using Titano.Protocol; using Titano.Protocol.Client; using Titano.Server.Storage; using Titano.Testing; +using UploadState = Titano.Data.Repositories.UploadState; namespace Titano.Server.Tests.Storage; @@ -85,8 +87,9 @@ public sealed class ArchiveModuleTests(TestContext context) public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace() { await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); - var (editor, _, _) = await server.UserAsync("alberto", "editor"); var photo = Photo(3); + // Room for the photo once: a repair puts back bytes the archive already counts. + var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: photo.Length + 1000); var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof"); var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId)); var path = Path.Combine(server.Options.ArchivePath, row.RelativePath); @@ -100,6 +103,50 @@ public sealed class ArchiveModuleTests(TestContext context) Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index)); Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path)); Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount); + using var me = await editor.GetAsync("/api/v1/auth/me"); + Assert.Equal(photo.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes); + } + + [Test] + public async Task AnIntactCopyElsewhereDoesNotStandForTheDamagedOneOfTheTargetSequence() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var photo = Photo(4); + var roof = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof"); + var path = Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(roof.FrameId)).RelativePath); + File.Delete(path); + // With the roof's copy gone, the same photo is filed into a second sequence. + var yard = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "yard"); + Assert.NotEqual(roof.SequenceId, yard.SequenceId); + + // Into the roof sequence the yard's copy is no shortcut, not even for a commit a crash interrupted: the bytes come + // and the roof's frame gets its file back. + var start = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", roof.SequenceId); + var id = Assert.NotNull(start.UploadId); + Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard")); + for (var i = 0; i * start.ChunkSize < photo.Length; i++) + { + using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(photo, start.ChunkSize, i)); + await put.ExpectAsync(HttpStatusCode.OK); + } + Assert.True(await server.Database.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: context.Cancellation)); + await server.Services.GetRequiredService().RecoverAsync(context.Cancellation); + using var complete = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null); + await complete.ExpectAsync(HttpStatusCode.OK); + var again = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse); + + Assert.Equal((roof.FrameId, roof.Index), (again.FrameId, again.Index)); + Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path)); + // Each sequence's own copy answers for it. + Assert.Equal((true, roof.FrameId), await ShortcutAsync(editor, photo, "roof")); + Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard")); + } + + private static async Task<(bool AlreadyExists, Guid? FrameId)> ShortcutAsync(HttpClient client, byte[] photo, string sessionKey) + { + var start = await Uploads.StartAsync(client, photo, "IMG_0001.JPG", sessionKey: sessionKey, expected: HttpStatusCode.OK); + return (start.AlreadyExists, start.FrameId); } private static byte[] Photo(int seed = 0)