fix(ingest): file uploads into API-made sequences and repair damaged frames
A sequence created with POST /api/v1/sequences had no folder, and the ingest looked for its sidecar at the archive root: every upload into it failed with 503 commit_failed, so the Uploader's "New sequence" target never worked. Its first frame now claims a folder by the rules of a sequence created by an upload (folder template with its title, unique variant, sidecar with the row's id, title, tags, rating, notes and project), and the row takes the folder in one transaction (DOCKER-1). A frame flagged missing or corrupt, or whose file is gone or has another size, no longer stands for an upload: the start short-circuit and crash recovery match only intact frames still on disk, and the ingest repairs such a frame with the verified copy (same id and position, flags cleared, a damaged file kept beside it as name.damaged-<time>) instead of calling it a duplicate. The Uploader deletes a source only after the server received and verified its bytes in that upload, never on "already on the server" (SRV-1). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
12 files changed
+441
-41
No files matched your search
@@ -14,7 +14,8 @@ namespace Titano.App.Import;
|
||||
/// server files them in one sequence — the same one when an interrupted import is started again. A typed session
|
||||
/// title is set on that sequence once. A file the server refuses (a hash mismatch, a format it rejects) is listed in
|
||||
/// <see cref="Report"/> and the import goes on (BRIEF §13: errors per file); a failed connection, sign-in or server
|
||||
/// stops it.
|
||||
/// stops it. A file the server already held counts as a duplicate: the server answers so only for a copy that is not
|
||||
/// flagged missing or damaged, and a server import never deletes anything from the card.
|
||||
/// </summary>
|
||||
public sealed class ServerImportSink(TitanoClient client, string stagingDirectory) : IImportSink
|
||||
{
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
@@ -22,7 +24,9 @@ internal sealed record SequenceTarget(string Key, string? Title, string? Preferr
|
||||
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
|
||||
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
|
||||
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
|
||||
/// SHA-256 the sequence already holds is reported as a duplicate and left in staging.
|
||||
/// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with
|
||||
/// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it
|
||||
/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame.
|
||||
/// </summary>
|
||||
public sealed class ArchiveIngest : IArchiveIngest
|
||||
{
|
||||
@@ -57,6 +61,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
{
|
||||
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
|
||||
if (AwaitsFolder(sequence, new ArchivePaths(archive.RootPath))) await PlaceAsync(archive, id, frame, cancellationToken).ConfigureAwait(false);
|
||||
sequenceId = id;
|
||||
}
|
||||
else
|
||||
@@ -110,12 +115,7 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
Created = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
var captured = CaptureTime(first);
|
||||
var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title));
|
||||
if (wanted.Length == 0) wanted = "Sequence";
|
||||
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var sequence = ClaimFolder(paths, wanted, taken, manifest);
|
||||
var sequence = await ClaimNewFolderAsync(archive, paths, target.PreferredFolder, first, manifest, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
@@ -134,6 +134,98 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
return manifest.Id;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A sequence created without photos (<c>POST /api/v1/sequences</c>, BRIEF §14.3) has no folder until its first frame.
|
||||
/// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence.
|
||||
/// </summary>
|
||||
internal static bool AwaitsFolder(Sequence sequence, ArchivePaths paths)
|
||||
{
|
||||
if (sequence.Folder.Length > 0) return false;
|
||||
var root = paths.Sequence(string.Empty);
|
||||
if (!File.Exists(root.ManifestPath)) return true;
|
||||
try
|
||||
{
|
||||
return ManifestStore.ReadSequence(root.ManifestPath).Id != sequence.Id;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException)
|
||||
{
|
||||
// An unreadable root sidecar: the commit reports it rather than placing the sequence elsewhere.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gives a sequence created without photos its folder, named from its first frame and title by the same rules as a
|
||||
/// sequence created by an upload, with a sidecar carrying the row's id, title, tags, rating and notes (and its project
|
||||
/// when one was saved); the index row takes the folder in one transaction. Serialised with the creation of sequences.
|
||||
/// </summary>
|
||||
private async Task PlaceAsync(Archive archive, Guid sequenceId, StagedFrame first, CancellationToken cancellationToken)
|
||||
{
|
||||
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
|
||||
// A concurrent first frame placed it meanwhile.
|
||||
if (!AwaitsFolder(row, paths)) return;
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var m = first.Metadata;
|
||||
var manifest = new SequenceManifest
|
||||
{
|
||||
Id = row.Id,
|
||||
Title = row.Title,
|
||||
// The photo names its camera; the creator's hint stands in when it does not.
|
||||
Camera = m.Make is null && m.Model is null && row.CameraModel is not null
|
||||
? new SidecarCamera { Model = row.CameraModel }
|
||||
: new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
|
||||
Lens = m.LensModel,
|
||||
Tags = row.Tags,
|
||||
Rating = row.Rating,
|
||||
Notes = row.Notes,
|
||||
Versions = new ManifestVersions { Settings = row.SettingsVersion, Frames = row.FramesVersion },
|
||||
CreatedBy = row.OwnerUserId?.ToString(),
|
||||
Created = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
var sequence = await ClaimNewFolderAsync(archive, paths, null, first, manifest, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (row.SettingsJson is { } settings) AtomicFile.WriteAllText(sequence.ProjectPath, settings);
|
||||
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
|
||||
var current = await tx.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Sequence {sequenceId} was deleted before its first frame.");
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, current), cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Nothing was filed into the folder yet: it goes, and the next first frame claims one again.
|
||||
if (File.Exists(sequence.ProjectPath)) File.Delete(sequence.ProjectPath);
|
||||
ReleaseFolder(sequence);
|
||||
throw;
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_creation.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A new folder for <paramref name="manifest"/>: the plan's folder or the archive's folder template, made unique.</summary>
|
||||
private async Task<SequencePaths> ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first,
|
||||
SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
var captured = CaptureTime(first);
|
||||
var wanted = preferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, manifest.Title));
|
||||
if (wanted.Length == 0) wanted = "Sequence";
|
||||
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
return ClaimFolder(paths, wanted, taken, manifest);
|
||||
}
|
||||
|
||||
/// <summary>The first free variant of <paramref name="wanted"/>, claimed by creating its sidecar (first writer wins).</summary>
|
||||
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet<string> taken, SequenceManifest manifest)
|
||||
{
|
||||
@@ -197,24 +289,36 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
var before = manifest.Frames.Count;
|
||||
var known = new Dictionary<string, SidecarFrame>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var f in manifest.Frames)
|
||||
{
|
||||
if (f.Sha256 is not null) known.TryAdd(f.Sha256, f);
|
||||
}
|
||||
var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
|
||||
// Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
|
||||
var filed = new Dictionary<string, Guid>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
|
||||
var repairs = new List<(Pending Pending, SidecarFrame Frame)>();
|
||||
var duplicates = new List<(Pending Pending, Guid FrameId)>();
|
||||
foreach (var p in batch)
|
||||
{
|
||||
if (known.TryGetValue(p.Frame.Sha256, out var existing))
|
||||
if (filed.TryGetValue(p.Frame.Sha256, out var same))
|
||||
{
|
||||
duplicates.Add((p, existing.Id));
|
||||
duplicates.Add((p, same));
|
||||
continue;
|
||||
}
|
||||
// Only a frame that still holds the content makes this copy a duplicate; a missing or damaged one is repaired with it.
|
||||
var holders = held[p.Frame.Sha256];
|
||||
if (holders.FirstOrDefault(f => IsIntact(sequence, f)) is { } intact)
|
||||
{
|
||||
duplicates.Add((p, intact.Id));
|
||||
continue;
|
||||
}
|
||||
if (holders.FirstOrDefault() is { } damaged)
|
||||
{
|
||||
filed[p.Frame.Sha256] = damaged.Id;
|
||||
repairs.Add((p, damaged));
|
||||
continue;
|
||||
}
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
|
||||
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
|
||||
known[p.Frame.Sha256] = frame;
|
||||
filed[p.Frame.Sha256] = frame.Id;
|
||||
accepted.Add((p, frame));
|
||||
}
|
||||
|
||||
@@ -235,17 +339,37 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
p.Completion.TrySetException(ex);
|
||||
}
|
||||
}
|
||||
var repaired = new HashSet<Guid>();
|
||||
foreach (var (p, frame) in repairs)
|
||||
{
|
||||
if (Repair(sequence, frame, p.Frame.StagedPath) is { } error) p.Completion.TrySetException(error);
|
||||
else repaired.Add(frame.Id);
|
||||
}
|
||||
var updated = SequenceFrames.Insert(manifest, moved);
|
||||
if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
if (repaired.Count > 0)
|
||||
{
|
||||
updated = SequenceFrames.Update(updated, repaired.ToDictionary(id => id,
|
||||
_ => (Func<SidecarFrame, SidecarFrame>)(f => f with { Flags = f.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt) })));
|
||||
}
|
||||
if (moved.Count > 0 || repaired.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
IngestJournal.Delete(sequence);
|
||||
|
||||
var byId = updated.Frames.ToDictionary(f => f.Id);
|
||||
if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false);
|
||||
if (moved.Count > 0 || repaired.Count > 0)
|
||||
{
|
||||
await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
|
||||
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
|
||||
foreach (var (p, frame) in repairs.Where(r => repaired.Contains(r.Frame.Id)))
|
||||
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
|
||||
var unrepaired = repairs.Select(r => r.Frame.Id).Where(id => !repaired.Contains(id)).ToHashSet();
|
||||
foreach (var (p, frameId) in duplicates)
|
||||
{
|
||||
if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
|
||||
// A copy of content this batch failed to file has nothing intact to stand for it.
|
||||
if (byId.TryGetValue(frameId, out var original) && !unrepaired.Contains(frameId))
|
||||
p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
|
||||
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
|
||||
}
|
||||
}
|
||||
@@ -284,11 +408,60 @@ public sealed class ArchiveIngest : IArchiveIngest
|
||||
return named;
|
||||
}
|
||||
|
||||
/// <summary>Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence.</summary>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
|
||||
CancellationToken cancellationToken)
|
||||
/// <summary>The frame still holds its content: not flagged missing or corrupt, and its file is there with its size.</summary>
|
||||
private static bool IsIntact(SequencePaths sequence, SidecarFrame frame)
|
||||
{
|
||||
var inStep = appended.All(f => f.I >= before)
|
||||
if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return false;
|
||||
var file = new FileInfo(sequence.File(frame.File));
|
||||
return file.Exists && file.Length == frame.Size;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Puts a verified copy where a damaged frame's file belongs. A file still there (corrupt, or another size) moves aside
|
||||
/// as <c>name.damaged-<time></c>, which no scan takes for a photo, and is never deleted (BRIEF §0.2 rule 8); it comes
|
||||
/// back when the copy cannot be moved in. Returns the failure, or null.
|
||||
/// </summary>
|
||||
private static Exception? Repair(SequencePaths sequence, SidecarFrame frame, string stagedPath)
|
||||
{
|
||||
var target = sequence.File(frame.File);
|
||||
string? aside = null;
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
if (File.Exists(target))
|
||||
{
|
||||
aside = string.Create(CultureInfo.InvariantCulture, $"{target}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
|
||||
File.Move(target, aside, overwrite: false);
|
||||
}
|
||||
File.Move(stagedPath, target, overwrite: false);
|
||||
Log.Info("archive", $"Repaired {sequence.ArchiveRelative(frame.File)} with a verified copy");
|
||||
return null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
if (aside is not null && !File.Exists(target) && File.Exists(aside))
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Move(aside, target);
|
||||
}
|
||||
catch (Exception again) when (again is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"{aside} stays beside {target}", again);
|
||||
}
|
||||
}
|
||||
return ex;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose
|
||||
/// flags change) resyncs the whole sequence.
|
||||
/// </summary>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
|
||||
bool repaired, CancellationToken cancellationToken)
|
||||
{
|
||||
var inStep = !repaired && appended.All(f => f.I >= before)
|
||||
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
|
||||
if (!inStep)
|
||||
{
|
||||
|
||||
@@ -17,7 +17,10 @@ public sealed record IngestRequest(
|
||||
DateTime? ClientCapturedAt,
|
||||
Guid? UserId);
|
||||
|
||||
/// <summary>The committed frame; <see cref="Duplicate"/> when the sequence already held a frame with that SHA-256.</summary>
|
||||
/// <summary>
|
||||
/// The committed frame; <see cref="Duplicate"/> when an intact frame of the sequence already held that SHA-256 (the file
|
||||
/// stays in staging). A frame with that SHA-256 that was missing or damaged is repaired with the file instead (not a duplicate).
|
||||
/// </summary>
|
||||
public sealed record IngestResult(Guid SequenceId, Guid FrameId, int Index, string RelativePath, bool Duplicate);
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -62,10 +62,18 @@ public sealed class FrameRepository
|
||||
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sequence_id = $1 AND idx > $2 ORDER BY idx LIMIT $3",
|
||||
[sequenceId, afterIndex, Cursor.ClampLimit(limit)], Map, cancellationToken);
|
||||
|
||||
/// <summary>Every frame with this content hash (duplicate detection, upload short-circuit).</summary>
|
||||
/// <summary>Every frame with this content hash (duplicate detection).</summary>
|
||||
public Task<IReadOnlyList<Frame>> FindBySha256Async(string sha256, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant()], Map, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Frames with this content hash that are not flagged missing or corrupt: only such a copy may stand for an upload
|
||||
/// (the upload short-circuit, a commit settled after a crash), so an upload of the same content repairs a damaged one.
|
||||
/// </summary>
|
||||
public Task<IReadOnlyList<Frame>> FindIntactBySha256Async(string sha256, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 AND (flags & $2) = 0 ORDER BY sequence_id, idx",
|
||||
[sha256.ToLowerInvariant(), (int)(FrameFlags.Missing | FrameFlags.Corrupt)], Map, cancellationToken);
|
||||
|
||||
/// <summary>Writes analysis results for many frames in one transaction; returns the number of frames updated.</summary>
|
||||
public Task<int> UpdateLightStatsAsync(IReadOnlyList<FrameLightStats> stats, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
|
||||
@@ -8,7 +8,8 @@ namespace Titano.Server.Storage;
|
||||
/// Upload housekeeping (BRIEF §14.4): at start, uploads a crash left half-committed are settled; then every half hour
|
||||
/// abandoned uploads expire, orphaned partial and staging files are removed, old upload records and dead tokens purged.
|
||||
/// </summary>
|
||||
public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, UploadStore store, TimeProvider time, ILogger<UploadMaintenance> logger)
|
||||
public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, ServerArchive archive, UploadStore store, TimeProvider time,
|
||||
ILogger<UploadMaintenance> logger)
|
||||
: BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan Interval = TimeSpan.FromMinutes(30);
|
||||
@@ -44,14 +45,16 @@ public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads left committing by a crash: committed when the frame with their hash exists, otherwise back to receiving
|
||||
/// (the partial file is intact, so the client simply completes again).
|
||||
/// Uploads left committing by a crash: committed when an intact frame with their hash exists, otherwise back to
|
||||
/// receiving (the partial file is intact, so the client simply completes again and the ingest files or repairs it).
|
||||
/// </summary>
|
||||
public async Task RecoverAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var upload in await db.Uploads.ListByStateAsync(UploadState.Committing, cancellationToken))
|
||||
{
|
||||
var frame = upload.JobId is null ? (await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault() : null;
|
||||
var frame = upload.JobId is null
|
||||
? (await db.Frames.FindIntactBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault(f => UploadRules.IsOnDisk(archive.Root, f))
|
||||
: null;
|
||||
if (frame is not null)
|
||||
{
|
||||
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, frame.Id, cancellationToken);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>Validation shared by frame and render uploads.</summary>
|
||||
@@ -30,6 +32,14 @@ public static class UploadRules
|
||||
return received;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The frame's original is in the archive with its recorded size. Together with flags that say neither missing nor
|
||||
/// corrupt (<c>FindIntactBySha256Async</c>), that is the copy an upload of the same content may stand for; a damaged
|
||||
/// one is repaired by uploading the content again.
|
||||
/// </summary>
|
||||
public static bool IsOnDisk(string archiveRoot, Frame frame) =>
|
||||
ArchivePathGuard.ExistingFile(archiveRoot, frame.RelativePath) is { } file && file.Length == frame.Size;
|
||||
|
||||
/// <summary>Free bytes on the volume holding <paramref name="path"/>, or null when the platform cannot tell.</summary>
|
||||
public static long? FreeBytes(string path)
|
||||
{
|
||||
|
||||
@@ -16,9 +16,10 @@ using UploadStateDto = Titano.Protocol.UploadState;
|
||||
namespace Titano.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256, resume an
|
||||
/// unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the file on the archive
|
||||
/// volume, verifies its hash and hands it to <see cref="IArchiveIngest"/>.
|
||||
/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an
|
||||
/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the
|
||||
/// file on the archive volume, verifies its hash and hands it to <see cref="IArchiveIngest"/>, which repairs a missing or
|
||||
/// damaged frame of that content.
|
||||
/// </summary>
|
||||
public sealed class UploadService(
|
||||
TitanoDatabase db,
|
||||
@@ -48,7 +49,8 @@ public sealed class UploadService(
|
||||
return Fail<StartUploadResponse>(404, "not_found", "sequence");
|
||||
|
||||
var sha = request.Sha256.ToLowerInvariant();
|
||||
var known = await db.Frames.FindBySha256Async(sha, cancellationToken);
|
||||
// Only a copy that is still whole stands for the upload: a client may delete its file on this answer.
|
||||
var known = (await db.Frames.FindIntactBySha256Async(sha, cancellationToken)).Where(f => UploadRules.IsOnDisk(archive.Root, f)).ToList();
|
||||
if (known.Count > 0)
|
||||
{
|
||||
var frame = known.FirstOrDefault(f => f.SequenceId == request.SequenceId) ?? known[0];
|
||||
|
||||
@@ -40,7 +40,8 @@ public interface IUploadEngine : INotifyPropertyChanged
|
||||
|
||||
/// <summary>
|
||||
/// Works through the <see cref="UploadQueue"/> with <see cref="Parallelism"/> concurrent files: hash, upload
|
||||
/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source the server confirmed.
|
||||
/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source whose bytes the server
|
||||
/// received and verified. A file the server already held is kept: no copy of it was verified on this run.
|
||||
/// Lives on the UI thread: state changes happen there while hashing and HTTP run on the thread pool.
|
||||
/// </summary>
|
||||
public sealed class UploadEngine : IUploadEngine, IDisposable
|
||||
@@ -221,7 +222,7 @@ public sealed class UploadEngine : IUploadEngine, IDisposable
|
||||
item.State = outcome == UploadOutcome.Verified ? UploadItemState.Verified : UploadItemState.Skipped;
|
||||
_queue.OnChanged();
|
||||
ItemConfirmed?.Invoke(this, item);
|
||||
if (_deleteAfterUpload()) DeleteConfirmedSource(item);
|
||||
if (outcome == UploadOutcome.Verified && _deleteAfterUpload()) DeleteConfirmedSource(item);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
@@ -312,8 +313,8 @@ public sealed class UploadEngine : IUploadEngine, IDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// BRIEF §0.2 rule 8: deletes only a file the server confirmed by hash, and only when it is still exactly the
|
||||
/// bytes that were hashed (same size and write time).
|
||||
/// BRIEF §0.2 rule 8: deletes only a file whose bytes the server received and verified by hash before committing
|
||||
/// them, and only when it is still exactly the bytes that were hashed (same size and write time).
|
||||
/// </summary>
|
||||
private static void DeleteConfirmedSource(UploadItem item)
|
||||
{
|
||||
|
||||
@@ -33,7 +33,7 @@ public sealed class UploadEngineTests(TestContext context)
|
||||
});
|
||||
|
||||
[Test]
|
||||
public void DeleteAfterUploadRemovesOnlyConfirmedUnchangedFiles() => SingleThreadedContext.Run(async () =>
|
||||
public void DeleteAfterUploadRemovesOnlyVerifiedUnchangedFiles() => SingleThreadedContext.Run(async () =>
|
||||
{
|
||||
var queue = Queue(UploadTargetMode.Sessions, 3);
|
||||
var (verified, existing, changed) = (queue.Items[0], queue.Items[1], queue.Items[2]);
|
||||
@@ -54,7 +54,8 @@ public sealed class UploadEngineTests(TestContext context)
|
||||
|
||||
Assert.Equal(UploadItemState.Skipped, existing.State);
|
||||
Assert.False(File.Exists(verified.Path));
|
||||
Assert.False(File.Exists(existing.Path));
|
||||
// The server matched its hash without receiving these bytes: no verified copy, so the source stays.
|
||||
Assert.True(File.Exists(existing.Path), "a file the server already held is kept");
|
||||
Assert.True(File.Exists(changed.Path), "a file changed after verification is kept");
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Data.Tests.Archiving.ArchiveFixture;
|
||||
@@ -121,4 +122,97 @@ public sealed class ArchiveIngestTests(TestContext context)
|
||||
Assert.FileMissing(sequence.IngestJournalPath);
|
||||
Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMissingFrameIsRepairedByTheSameContentInsteadOfCountingAsDuplicate()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
||||
var row = (await fx.Db.Sequences.GetAsync(original.SequenceId))!;
|
||||
var file = Path.Combine(fx.Archive.RootPath, original.RelativePath);
|
||||
var bytes = await File.ReadAllBytesAsync(file, fx.Ct);
|
||||
File.Delete(file);
|
||||
await fx.Service.Indexer.ReconcileAsync(fx.Archive, row, fx.Ct);
|
||||
Assert.Equal(FrameFlags.Missing, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
||||
|
||||
var again = fx.Stage(Start, 7);
|
||||
var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
|
||||
|
||||
Assert.False(repaired.Duplicate, "the server holds no intact copy: this one is filed");
|
||||
Assert.Equal((original.FrameId, original.Index, original.RelativePath), (repaired.FrameId, repaired.Index, repaired.RelativePath));
|
||||
Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
|
||||
Assert.FileMissing(again.Path);
|
||||
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
||||
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
|
||||
Assert.Equal(FrameFlags.None, manifest.Frames.Single().Flags);
|
||||
Assert.False((await fx.Db.Sequences.GetAsync(original.SequenceId))!.Status.HasFlag(SequenceStatus.HasErrors));
|
||||
|
||||
// Intact again: the next copy is a duplicate and stays in staging.
|
||||
var third = fx.Stage(Start, 7);
|
||||
Assert.True((await fx.Service.Ingest.CommitAsync(Request(fx, third), fx.Ct)).Duplicate);
|
||||
Assert.FileExists(third.Path);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACorruptFrameIsRepairedAndItsDamagedFileKeptAside()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
||||
var file = Path.Combine(fx.Archive.RootPath, original.RelativePath);
|
||||
var bytes = await File.ReadAllBytesAsync(file, fx.Ct);
|
||||
var damaged = bytes.ToArray();
|
||||
damaged[^10] ^= 0xFF;
|
||||
await File.WriteAllBytesAsync(file, damaged, fx.Ct);
|
||||
await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
|
||||
Assert.Equal(FrameFlags.Corrupt, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
||||
|
||||
var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
|
||||
|
||||
Assert.False(repaired.Duplicate);
|
||||
Assert.Equal(original.FrameId, repaired.FrameId);
|
||||
Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
|
||||
var aside = Directory.GetFiles(Path.GetDirectoryName(file)!, Path.GetFileName(file) + ".damaged-*");
|
||||
Assert.Count(1, aside);
|
||||
Assert.SequenceEqual(damaged, await File.ReadAllBytesAsync(aside[0], fx.Ct));
|
||||
Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
|
||||
// The kept file is no photo to a scan: the sequence still has one frame.
|
||||
Assert.Equal(0, (await fx.ScanAsync(ScanMode.Full)).Failed.Count);
|
||||
Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ASequenceCreatedWithoutPhotosGetsItsFolderWithTheFirstFrame()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
// As POST /api/v1/sequences creates it: a row without a folder.
|
||||
var created = new Sequence
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
ArchiveId = fx.Archive.Id,
|
||||
Title = "Harbour sunset",
|
||||
Folder = string.Empty,
|
||||
Tags = ["harbour"],
|
||||
Rating = 4,
|
||||
Origin = SequenceOrigin.Server,
|
||||
OwnerUserId = Guid.CreateVersion7(),
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
await fx.Db.Sequences.UpsertAsync(created, fx.Ct);
|
||||
|
||||
var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: created.Id), fx.Ct);
|
||||
var second = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), sequenceId: created.Id, name: "IMG_0002.JPG"), fx.Ct);
|
||||
|
||||
Assert.Equal((created.Id, created.Id), (first.SequenceId, second.SequenceId));
|
||||
var row = (await fx.Db.Sequences.GetAsync(created.Id))!;
|
||||
Assert.Equal("2026/2026-09-28 Canon EOS R5", row.Folder);
|
||||
Assert.Equal(("Harbour sunset", 4, SequenceOrigin.Server, created.OwnerUserId, 2), (row.Title, row.Rating, row.Origin, row.OwnerUserId, row.FrameCount));
|
||||
Assert.SequenceEqual(["harbour"], row.Tags);
|
||||
Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", first.RelativePath);
|
||||
var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
|
||||
Assert.Equal((created.Id, "Harbour sunset", created.OwnerUserId.ToString()), (manifest.Id, manifest.Title, manifest.CreatedBy));
|
||||
Assert.SequenceEqual([first.FrameId, second.FrameId], manifest.Frames.Select(f => f.Id));
|
||||
Assert.FileMissing(fx.Paths.Sequence(string.Empty).ManifestPath);
|
||||
// The index can be rebuilt from the sidecar it now has.
|
||||
Assert.Equal(0, (await fx.ScanAsync()).Indexed);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
using System.Net;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
using Titano.Server.Storage;
|
||||
using Titano.Testing;
|
||||
|
||||
@@ -42,11 +44,69 @@ public sealed class ArchiveModuleTests(TestContext context)
|
||||
}
|
||||
}
|
||||
|
||||
private static byte[] Photo()
|
||||
[Test]
|
||||
public async Task ASequenceCreatedThroughTheApiTakesUploadsIntoAFolderOfItsOwn()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
|
||||
var (_, _, token) = await server.UserAsync("alberto", "editor");
|
||||
var source = context.CreateTempDirectory("source");
|
||||
var photos = new List<(string Path, byte[] Bytes)>();
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
var path = Path.Combine(source, $"IMG_000{i + 1}.JPG");
|
||||
var bytes = Photo(i + 1);
|
||||
await File.WriteAllBytesAsync(path, bytes);
|
||||
photos.Add((path, bytes));
|
||||
}
|
||||
|
||||
// The Uploader's "New sequence" target: create it, then upload into it (BRIEF §15).
|
||||
using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "Titano Uploader (tests)") { Token = token };
|
||||
var created = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null), context.Cancellation);
|
||||
var first = Assert.NotNull(await client.UploadFileAsync(photos[0].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 11), ct: context.Cancellation));
|
||||
var second = Assert.NotNull(await client.UploadFileAsync(photos[1].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 21), ct: context.Cancellation));
|
||||
|
||||
Assert.Equal((created.Id, created.Id, 0, 1), (first.SequenceId, second.SequenceId, first.Index, second.Index));
|
||||
var row = Assert.NotNull(await server.Database.Sequences.GetAsync(created.Id));
|
||||
Assert.True(row.Folder.StartsWith("2026/2026-09-28", StringComparison.Ordinal), row.Folder);
|
||||
Assert.Equal(("Harbour sunset", 2), (row.Title, row.FrameCount));
|
||||
var sidecar = ManifestStore.ReadSequence(new ArchivePaths(server.Options.ArchivePath).Sequence(row.Folder).ManifestPath);
|
||||
Assert.Equal((created.Id, "Harbour sunset"), (sidecar.Id, sidecar.Title));
|
||||
Assert.SequenceEqual([first.FrameId, second.FrameId], sidecar.Frames.Select(f => f.Id));
|
||||
Assert.FileMissing(Path.Combine(server.Options.ArchivePath, ".titano", "sequence.json"));
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
using var downloaded = new MemoryStream();
|
||||
await client.DownloadFrameAsync(i == 0 ? first.FrameId : second.FrameId, "original", downloaded, ct: context.Cancellation);
|
||||
Assert.BytesEqual(photos[i].Bytes, downloaded.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var photo = Photo(3);
|
||||
var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
|
||||
var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
|
||||
var path = Path.Combine(server.Options.ArchivePath, row.RelativePath);
|
||||
var known = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof", expected: HttpStatusCode.OK);
|
||||
Assert.Equal((true, frame.FrameId), (known.AlreadyExists, known.FrameId));
|
||||
|
||||
File.Delete(path);
|
||||
// No "already exists" for a copy that is gone: the bytes come again and the frame gets its file back.
|
||||
var again = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
|
||||
|
||||
Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index));
|
||||
Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
|
||||
Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount);
|
||||
}
|
||||
|
||||
private static byte[] Photo(int seed = 0)
|
||||
{
|
||||
const int width = 96, height = 64;
|
||||
var pixels = new byte[width * height * 3];
|
||||
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251);
|
||||
for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
|
||||
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Protocol;
|
||||
using Titano.Protocol.Client;
|
||||
using Titano.Server.Storage;
|
||||
@@ -96,6 +97,49 @@ public sealed class UploadTests(TestContext context)
|
||||
Assert.Equal(frame.FrameId, again.FrameId);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACopyFlaggedOrGoneIsNoShortcut()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
|
||||
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
|
||||
// A client may delete its file on "already exists": a damaged copy must ask for the bytes again.
|
||||
var flagged = Uploads.Photo(5000, 11);
|
||||
var first = await Uploads.UploadAsync(uploader, flagged);
|
||||
await server.Database.Frames.UpdateFlagsAsync([first.FrameId], FrameFlags.Corrupt, FrameFlags.None);
|
||||
Assert.False((await Uploads.StartAsync(uploader, flagged)).AlreadyExists);
|
||||
|
||||
var gone = Uploads.Photo(5000, 12);
|
||||
var second = await Uploads.UploadAsync(uploader, gone);
|
||||
File.Delete(Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(second.FrameId)).RelativePath));
|
||||
Assert.False((await Uploads.StartAsync(uploader, gone)).AlreadyExists);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACommitLeftByACrashIsSettledOnlyByAnIntactFrame()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
|
||||
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
|
||||
var data = Uploads.Photo(Chunk + 100, 13);
|
||||
var missing = await Uploads.UploadAsync(uploader, data);
|
||||
await server.Database.Frames.UpdateFlagsAsync([missing.FrameId], FrameFlags.Missing, FrameFlags.None);
|
||||
var start = await Uploads.StartAsync(uploader, data);
|
||||
var id = Assert.NotNull(start.UploadId);
|
||||
for (var i = 0; i < 2; i++)
|
||||
{
|
||||
using var put = await Uploads.PutChunkAsync(uploader, id, i, Uploads.Chunk(data, Chunk, i));
|
||||
await put.ExpectAsync(HttpStatusCode.OK);
|
||||
}
|
||||
// The server stopped while committing it.
|
||||
Assert.True(await server.Database.Uploads.TryTransitionAsync(id, Titano.Data.Repositories.UploadState.Receiving,
|
||||
Titano.Data.Repositories.UploadState.Committing, cancellationToken: context.Cancellation));
|
||||
|
||||
await server.Services.GetRequiredService<UploadMaintenance>().RecoverAsync(context.Cancellation);
|
||||
|
||||
// The flagged frame does not stand for it: the client completes again with its part file intact.
|
||||
using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
|
||||
Assert.True(server.Services.GetRequiredService<UploadStore>().Exists(id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadKeepsItsChunkSizeWhenTheSettingShrinks()
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user