fix(upload): repairs charge no quota and another sequence's copy no longer blocks them

A re-upload that repairs a missing or damaged frame is now reported as
IngestResult.Repaired and adds nothing to the uploader's used bytes; the
quota checks at start and completion skip uploads whose target sequence
already holds the content. The start shortcut and crash recovery share
one rule (UploadRules.StandIn): an intact copy stands for the upload, the
target sequence's own copy first, except when the target holds the
content only in damaged copies, which the upload's bytes then repair.
A sequence adopted at the archive root that holds frames is never given
a new folder when its root sidecar is missing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-12 10:53:49 +00:00
1 parent bb55e47b47
commit 886132b5bb
8 files changed
+150 -39

No files matched your search

+4 -3
View File
@@ -136,11 +136,12 @@ public sealed class ArchiveIngest : IArchiveIngest
/// <summary>
/// A sequence created without photos (<c>POST /api/v1/sequences</c>, BRIEF §14.3) has no folder until its first frame.
/// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence.
/// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence; one
/// that holds frames is never moved elsewhere, even while its root sidecar is missing.
/// </summary>
internal static bool AwaitsFolder(Sequence sequence, ArchivePaths paths)
{
if (sequence.Folder.Length > 0) return false;
if (sequence.Folder.Length > 0 || sequence.FrameCount > 0) return false;
var root = paths.Sequence(string.Empty);
if (!File.Exists(root.ManifestPath)) return true;
try
@@ -363,7 +364,7 @@ public sealed class ArchiveIngest : IArchiveIngest
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
foreach (var (p, frame) in repairs.Where(r => repaired.Contains(r.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false) with { Repaired = true });
var unrepaired = repairs.Select(r => r.Frame.Id).Where(id => !repaired.Contains(id)).ToHashSet();
foreach (var (p, frameId) in duplicates)
{
+3 -2
View File
@@ -19,9 +19,10 @@ public sealed record IngestRequest(
/// <summary>
/// The committed frame; <see cref="Duplicate"/> when an intact frame of the sequence already held that SHA-256 (the file
/// stays in staging). A frame with that SHA-256 that was missing or damaged is repaired with the file instead (not a duplicate).
/// stays in staging). A frame with that SHA-256 that was missing or damaged is repaired with the file instead:
/// <see cref="Repaired"/>, not a duplicate, and the archive holds no more bytes than before.
/// </summary>
public sealed record IngestResult(Guid SequenceId, Guid FrameId, int Index, string RelativePath, bool Duplicate);
public sealed record IngestResult(Guid SequenceId, Guid FrameId, int Index, string RelativePath, bool Duplicate, bool Repaired = false);
/// <summary>
/// Commits staged files into the archive layout (originals folder, naming templates, <c>sequence.json</c> written
@@ -65,18 +65,10 @@ public sealed class FrameRepository
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sequence_id = $1 AND idx > $2 ORDER BY idx LIMIT $3",
[sequenceId, afterIndex, Cursor.ClampLimit(limit)], Map, cancellationToken);
/// <summary>Every frame with this content hash (duplicate detection).</summary>
/// <summary>Every frame with this content hash (duplicate detection, upload short-circuit).</summary>
public Task<IReadOnlyList<Frame>> FindBySha256Async(string sha256, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant()], Map, cancellationToken);
/// <summary>
/// Frames with this content hash that are not flagged missing or corrupt: only such a copy may stand for an upload
/// (the upload short-circuit, a commit settled after a crash), so an upload of the same content repairs a damaged one.
/// </summary>
public Task<IReadOnlyList<Frame>> FindIntactBySha256Async(string sha256, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 AND (flags & $2) = 0 ORDER BY sequence_id, idx",
[sha256.ToLowerInvariant(), (int)(FrameFlags.Missing | FrameFlags.Corrupt)], Map, cancellationToken);
/// <summary>Id and index of every frame of a sequence, in index order.</summary>
public Task<IReadOnlyList<FramePosition>> ListPositionsAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ListAsync("SELECT id, idx FROM frames WHERE sequence_id = $1 ORDER BY idx", [sequenceId], r => new FramePosition(r.GetGuid(0), r.GetInt32(1)),
+14 -5
View File
@@ -1,3 +1,4 @@
using Titano.Core.Model;
using Titano.Data;
using Titano.Data.Repositories;
using Titano.Server.Configuration;
@@ -45,16 +46,15 @@ public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options,
}
/// <summary>
/// Uploads left committing by a crash: committed when an intact frame with their hash exists, otherwise back to
/// receiving (the partial file is intact, so the client simply completes again and the ingest files or repairs it).
/// Uploads left committing by a crash: committed when an intact frame stands for their content as at the start of an
/// upload (<see cref="UploadRules.StandIn"/>), otherwise back to receiving (the partial file is intact, so the client
/// simply completes again and the ingest files or repairs it).
/// </summary>
public async Task RecoverAsync(CancellationToken cancellationToken)
{
foreach (var upload in await db.Uploads.ListByStateAsync(UploadState.Committing, cancellationToken))
{
var frame = upload.JobId is null
? (await db.Frames.FindIntactBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault(f => UploadRules.IsOnDisk(archive.Root, f))
: null;
var frame = upload.JobId is null ? await StandInAsync(upload, cancellationToken) : null;
if (frame is not null)
{
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, frame.Id, cancellationToken);
@@ -68,6 +68,15 @@ public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options,
}
}
/// <summary>The intact frame that settles an interrupted commit, or null when the client must complete it again.</summary>
private async Task<Frame?> StandInAsync(Upload upload, CancellationToken cancellationToken)
{
var copies = await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken);
if (copies.Count == 0) return null;
var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken);
return UploadRules.StandIn(archive.Root, copies, target);
}
public async Task SweepAsync(CancellationToken cancellationToken)
{
var now = time.GetUtcNow();
+32 -5
View File
@@ -1,4 +1,5 @@
using Titano.Core.Model;
using Titano.Data;
namespace Titano.Server.Storage;
@@ -32,14 +33,40 @@ public static class UploadRules
return received;
}
/// <summary>
/// The frame's original is in the archive with its recorded size. Together with flags that say neither missing nor
/// corrupt (<c>FindIntactBySha256Async</c>), that is the copy an upload of the same content may stand for; a damaged
/// one is repaired by uploading the content again.
/// </summary>
/// <summary>The frame's original is in the archive with its recorded size.</summary>
public static bool IsOnDisk(string archiveRoot, Frame frame) =>
ArchivePathGuard.ExistingFile(archiveRoot, frame.RelativePath) is { } file && file.Length == frame.Size;
/// <summary>
/// Not flagged missing or corrupt, and <see cref="IsOnDisk"/>: the copy holds its content, so it may stand for an upload
/// of the same content; a damaged one is repaired by uploading the content again.
/// </summary>
public static bool IsIntact(string archiveRoot, Frame frame) =>
(frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0 && IsOnDisk(archiveRoot, frame);
/// <summary>
/// The frame that stands for an upload of content the archive already has (<paramref name="copies"/>): an intact copy,
/// the one in the upload's <paramref name="target"/> sequence first. Null when there is none, or when the target holds
/// the content only in damaged copies: the upload's bytes repair them, whatever other sequences hold.
/// </summary>
public static Frame? StandIn(string archiveRoot, IReadOnlyList<Frame> copies, Guid? target)
{
var intact = copies.Where(f => IsIntact(archiveRoot, f)).ToList();
if (copies.Any(f => f.SequenceId == target) && !intact.Any(f => f.SequenceId == target)) return null;
return intact.FirstOrDefault(f => f.SequenceId == target) ?? intact.FirstOrDefault();
}
/// <summary>
/// The sequence an upload files into: its own, or the one its session key names (the user's own key without one);
/// null while that sequence does not exist yet.
/// </summary>
public static async Task<Guid?> TargetAsync(TitanoDatabase db, Guid archiveId, Guid? sequenceId, string? sessionKey, Guid userId,
CancellationToken cancellationToken) =>
sequenceId ?? await db.Sequences.FindBySessionKeyAsync(archiveId, SessionKeyOf(sessionKey, userId), cancellationToken).ConfigureAwait(false);
/// <summary>The session key an upload without a sequence is grouped by: the client's, or one per user.</summary>
public static string SessionKeyOf(string? sessionKey, Guid userId) => sessionKey ?? "user-" + userId.ToString("N");
/// <summary>Free bytes on the volume holding <paramref name="path"/>, or null when the platform cannot tell.</summary>
public static long? FreeBytes(string path)
{
+25 -13
View File
@@ -19,7 +19,7 @@ namespace Titano.Server.Storage;
/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an
/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the
/// file on the archive volume, verifies its hash and hands it to <see cref="IArchiveIngest"/>, which repairs a missing or
/// damaged frame of that content.
/// damaged frame of that content (no quota is charged for a repair: the archive already counts those bytes).
/// </summary>
public sealed class UploadService(
TitanoDatabase db,
@@ -45,23 +45,23 @@ public sealed class UploadService(
if (request.Size <= 0) return Fail<StartUploadResponse>(400, "bad_request", "size");
if (request.Size > options.MaxUploadBytes) return Fail<StartUploadResponse>(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
if (request.SessionKey is { Length: > 200 }) return Fail<StartUploadResponse>(400, "bad_request", "sessionKey");
if (request.SequenceId is { } target && await db.Sequences.GetAsync(target, cancellationToken) is null)
if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null)
return Fail<StartUploadResponse>(404, "not_found", "sequence");
var sha = request.Sha256.ToLowerInvariant();
var sessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim();
var copies = await db.Frames.FindBySha256Async(sha, cancellationToken);
var target = copies.Count == 0 ? null : await UploadRules.TargetAsync(db, archive.Id, request.SequenceId, sessionKey, caller.Id, cancellationToken);
// Only a copy that is still whole stands for the upload: a client may delete its file on this answer.
var known = (await db.Frames.FindIntactBySha256Async(sha, cancellationToken)).Where(f => UploadRules.IsOnDisk(archive.Root, f)).ToList();
if (known.Count > 0)
{
var frame = known.FirstOrDefault(f => f.SequenceId == request.SequenceId) ?? known[0];
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, frame.Id);
}
if (UploadRules.StandIn(archive.Root, copies, target) is { } known)
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id);
var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken);
if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id))
return Started(resumable);
if (caller.User.QuotaBytes > 0)
// A repair puts back bytes the archive already counts: no quota for it.
if (caller.User.QuotaBytes > 0 && !copies.Any(f => f.SequenceId == target))
{
var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken);
if (caller.User.UsedBytes + pending + request.Size > caller.User.QuotaBytes)
@@ -84,7 +84,7 @@ public sealed class UploadService(
State = UploadState.Receiving,
Created = now,
UpdatedAt = now,
SessionKey = string.IsNullOrWhiteSpace(request.SessionKey) ? null : request.SessionKey.Trim(),
SessionKey = sessionKey,
}, cancellationToken);
store.Create(upload.Id, upload.Size);
return Started(upload);
@@ -159,7 +159,7 @@ public sealed class UploadService(
return Fail<CompleteUploadResponse>(409, "upload_state");
var user = await db.Users.GetAsync(caller.Id, stopping);
if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes)
if (user is { QuotaBytes: > 0 } && user.UsedBytes + upload.Size > user.QuotaBytes && !await HeldByTargetAsync(upload, stopping))
{
await FailAsync(upload);
return Fail<CompleteUploadResponse>(403, "quota_exceeded", user.QuotaBytes.ToString(CultureInfo.InvariantCulture));
@@ -175,7 +175,7 @@ public sealed class UploadService(
archive.Id,
upload.SequenceId,
NewSequenceTitle: null,
upload.SequenceId is null ? upload.SessionKey ?? "user-" + caller.Id.ToString("N") : null,
upload.SequenceId is null ? UploadRules.SessionKeyOf(upload.SessionKey, caller.Id) : null,
staged,
upload.FileName,
upload.Size,
@@ -195,7 +195,7 @@ public sealed class UploadService(
await db.Uploads.TryTransitionAsync(id, UploadState.Committing, UploadState.Committed, result.FrameId, CancellationToken.None);
store.Delete(id);
if (File.Exists(staged)) UploadStore.TryDelete(staged);
if (!result.Duplicate) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None);
if (!result.Duplicate && !result.Repaired) await db.Users.AddUsedBytesAsync(caller.Id, upload.Size, CancellationToken.None);
if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } sequence)
{
await sequenceJobs.FramesAddedAsync(sequence, CancellationToken.None);
@@ -235,6 +235,18 @@ public sealed class UploadService(
store.Delete(upload.Id);
}
/// <summary>
/// The sequence the upload files into holds its content already: filing it adds no bytes (an intact copy makes it a
/// duplicate, a damaged one is repaired with it).
/// </summary>
private async Task<bool> HeldByTargetAsync(Upload upload, CancellationToken cancellationToken)
{
var copies = await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken);
if (copies.Count == 0) return false;
var target = await UploadRules.TargetAsync(db, archive.Id, upload.SequenceId, upload.SessionKey, upload.UserId, cancellationToken);
return copies.Any(f => f.SequenceId == target);
}
private StartUploadResponse Started(Upload upload) =>
new(upload.Id, upload.ChunkSize, UploadRules.ReceivedChunks(upload), AlreadyExists: false, FrameId: null);
@@ -143,6 +143,7 @@ public sealed class ArchiveIngestTests(TestContext context)
var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
Assert.False(repaired.Duplicate, "the server holds no intact copy: this one is filed");
Assert.Equal((false, true), (original.Repaired, repaired.Repaired));
Assert.Equal((original.FrameId, original.Index, original.RelativePath), (repaired.FrameId, repaired.Index, repaired.RelativePath));
Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
Assert.FileMissing(again.Path);
@@ -172,7 +173,7 @@ public sealed class ArchiveIngestTests(TestContext context)
var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
Assert.False(repaired.Duplicate);
Assert.Equal((false, true), (repaired.Duplicate, repaired.Repaired));
Assert.Equal(original.FrameId, repaired.FrameId);
Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
var aside = Directory.GetFiles(Path.GetDirectoryName(file)!, Path.GetFileName(file) + ".damaged-*");
@@ -220,6 +221,27 @@ public sealed class ArchiveIngestTests(TestContext context)
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
}
[Test]
public async Task ASequenceAdoptedAtTheRootIsNeverMovedWhenItsSidecarIsGone()
{
await using var fx = await CreateAsync(context);
Photos.WriteSeries(fx.Archive.RootPath, Start, 2);
await fx.ScanAsync();
var adopted = Assert.NotNull((await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id, fx.Ct)).SingleOrDefault());
Assert.Equal(string.Empty, adopted.Folder);
File.Delete(fx.Paths.Sequence(string.Empty).ManifestPath);
// It held photos, so it is no sequence waiting for a folder: the commit reports the missing sidecar.
var staged = fx.Stage(Start.AddMinutes(5), 3);
await Assert.ThrowsAsync<IOException>(() => fx.Service.Ingest.CommitAsync(Request(fx, staged, sequenceId: adopted.Id), fx.Ct));
var row = (await fx.Db.Sequences.GetAsync(adopted.Id))!;
Assert.Equal((string.Empty, 2), (row.Folder, row.FrameCount));
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(adopted.Id));
Assert.False(Directory.Exists(Path.Combine(fx.Archive.RootPath, "2026")), "no folder was claimed for it");
Assert.FileExists(staged.Path);
}
[Test]
public async Task FramesInsertedBeforeOthersTakeTheProjectPositionsAlong()
{
@@ -1,10 +1,12 @@
using System.Net;
using Microsoft.Extensions.DependencyInjection;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Storage;
using Titano.Testing;
using UploadState = Titano.Data.Repositories.UploadState;
namespace Titano.Server.Tests.Storage;
@@ -85,8 +87,9 @@ public sealed class ArchiveModuleTests(TestContext context)
public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo(3);
// Room for the photo once: a repair puts back bytes the archive already counts.
var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: photo.Length + 1000);
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
var path = Path.Combine(server.Options.ArchivePath, row.RelativePath);
@@ -100,6 +103,50 @@ public sealed class ArchiveModuleTests(TestContext context)
Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount);
using var me = await editor.GetAsync("/api/v1/auth/me");
Assert.Equal(photo.Length, (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes);
}
[Test]
public async Task AnIntactCopyElsewhereDoesNotStandForTheDamagedOneOfTheTargetSequence()
{
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
var (editor, _, _) = await server.UserAsync("alberto", "editor");
var photo = Photo(4);
var roof = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
var path = Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(roof.FrameId)).RelativePath);
File.Delete(path);
// With the roof's copy gone, the same photo is filed into a second sequence.
var yard = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "yard");
Assert.NotEqual(roof.SequenceId, yard.SequenceId);
// Into the roof sequence the yard's copy is no shortcut, not even for a commit a crash interrupted: the bytes come
// and the roof's frame gets its file back.
var start = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", roof.SequenceId);
var id = Assert.NotNull(start.UploadId);
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
for (var i = 0; i * start.ChunkSize < photo.Length; i++)
{
using var put = await Uploads.PutChunkAsync(editor, id, i, Uploads.Chunk(photo, start.ChunkSize, i));
await put.ExpectAsync(HttpStatusCode.OK);
}
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, UploadState.Receiving, UploadState.Committing, cancellationToken: context.Cancellation));
await server.Services.GetRequiredService<UploadMaintenance>().RecoverAsync(context.Cancellation);
using var complete = await editor.PostAsync($"/api/v1/uploads/{id}/complete", null);
await complete.ExpectAsync(HttpStatusCode.OK);
var again = await complete.ReadAsync(ProtocolJson.Default.CompleteUploadResponse);
Assert.Equal((roof.FrameId, roof.Index), (again.FrameId, again.Index));
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
// Each sequence's own copy answers for it.
Assert.Equal((true, roof.FrameId), await ShortcutAsync(editor, photo, "roof"));
Assert.Equal((true, yard.FrameId), await ShortcutAsync(editor, photo, "yard"));
}
private static async Task<(bool AlreadyExists, Guid? FrameId)> ShortcutAsync(HttpClient client, byte[] photo, string sessionKey)
{
var start = await Uploads.StartAsync(client, photo, "IMG_0001.JPG", sessionKey: sessionKey, expected: HttpStatusCode.OK);
return (start.AlreadyExists, start.FrameId);
}
private static byte[] Photo(int seed = 0)