diff --git a/src/Titano.App/Import/ServerImportSink.cs b/src/Titano.App/Import/ServerImportSink.cs index d65b352e..b0feb616 100644 --- a/src/Titano.App/Import/ServerImportSink.cs +++ b/src/Titano.App/Import/ServerImportSink.cs @@ -14,7 +14,8 @@ namespace Titano.App.Import; /// server files them in one sequence — the same one when an interrupted import is started again. A typed session /// title is set on that sequence once. A file the server refuses (a hash mismatch, a format it rejects) is listed in /// and the import goes on (BRIEF §13: errors per file); a failed connection, sign-in or server -/// stops it. +/// stops it. A file the server already held counts as a duplicate: the server answers so only for a copy that is not +/// flagged missing or damaged, and a server import never deletes anything from the card. /// public sealed class ServerImportSink(TitanoClient client, string stagingDirectory) : IImportSink { diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs index 29a31c2a..c23f1b86 100644 --- a/src/Titano.Data/Archiving/ArchiveIngest.cs +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -1,6 +1,8 @@ using System.Collections.Concurrent; using System.Globalization; using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.IO; using Titano.Core.Localization; using Titano.Core.Metadata; using Titano.Core.Model; @@ -22,7 +24,9 @@ internal sealed record SequenceTarget(string Key, string? Title, string? Preferr /// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits /// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps /// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose -/// SHA-256 the sequence already holds is reported as a duplicate and left in staging. +/// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with +/// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it +/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame. /// public sealed class ArchiveIngest : IArchiveIngest { @@ -57,6 +61,7 @@ public sealed class ArchiveIngest : IArchiveIngest { var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false); if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request)); + if (AwaitsFolder(sequence, new ArchivePaths(archive.RootPath))) await PlaceAsync(archive, id, frame, cancellationToken).ConfigureAwait(false); sequenceId = id; } else @@ -110,12 +115,7 @@ public sealed class ArchiveIngest : IArchiveIngest Created = now, UpdatedAt = now, }; - var captured = CaptureTime(first); - var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title)); - if (wanted.Length == 0) wanted = "Sequence"; - var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) - .Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase); - var sequence = ClaimFolder(paths, wanted, taken, manifest); + var sequence = await ClaimNewFolderAsync(archive, paths, target.PreferredFolder, first, manifest, cancellationToken).ConfigureAwait(false); await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) { @@ -134,6 +134,98 @@ public sealed class ArchiveIngest : IArchiveIngest return manifest.Id; } + /// + /// A sequence created without photos (POST /api/v1/sequences, BRIEF §14.3) has no folder until its first frame. + /// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence. + /// + internal static bool AwaitsFolder(Sequence sequence, ArchivePaths paths) + { + if (sequence.Folder.Length > 0) return false; + var root = paths.Sequence(string.Empty); + if (!File.Exists(root.ManifestPath)) return true; + try + { + return ManifestStore.ReadSequence(root.ManifestPath).Id != sequence.Id; + } + catch (Exception ex) when (ex is IOException or InvalidDataException) + { + // An unreadable root sidecar: the commit reports it rather than placing the sequence elsewhere. + return false; + } + } + + /// + /// Gives a sequence created without photos its folder, named from its first frame and title by the same rules as a + /// sequence created by an upload, with a sidecar carrying the row's id, title, tags, rating and notes (and its project + /// when one was saved); the index row takes the folder in one transaction. Serialised with the creation of sequences. + /// + private async Task PlaceAsync(Archive archive, Guid sequenceId, StagedFrame first, CancellationToken cancellationToken) + { + await _creation.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var paths = new ArchivePaths(archive.RootPath); + var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed."); + // A concurrent first frame placed it meanwhile. + if (!AwaitsFolder(row, paths)) return; + ArchiveVolume.EnsureWritable(paths); + var now = DateTimeOffset.UtcNow; + var m = first.Metadata; + var manifest = new SequenceManifest + { + Id = row.Id, + Title = row.Title, + // The photo names its camera; the creator's hint stands in when it does not. + Camera = m.Make is null && m.Model is null && row.CameraModel is not null + ? new SidecarCamera { Model = row.CameraModel } + : new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial }, + Lens = m.LensModel, + Tags = row.Tags, + Rating = row.Rating, + Notes = row.Notes, + Versions = new ManifestVersions { Settings = row.SettingsVersion, Frames = row.FramesVersion }, + CreatedBy = row.OwnerUserId?.ToString(), + Created = now, + UpdatedAt = now, + }; + var sequence = await ClaimNewFolderAsync(archive, paths, null, first, manifest, cancellationToken).ConfigureAwait(false); + try + { + if (row.SettingsJson is { } settings) AtomicFile.WriteAllText(sequence.ProjectPath, settings); + await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false); + var current = await tx.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Sequence {sequenceId} was deleted before its first frame."); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, current), cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + // Nothing was filed into the folder yet: it goes, and the next first frame claims one again. + if (File.Exists(sequence.ProjectPath)) File.Delete(sequence.ProjectPath); + ReleaseFolder(sequence); + throw; + } + await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + finally + { + _creation.Release(); + } + } + + /// A new folder for : the plan's folder or the archive's folder template, made unique. + private async Task ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first, + SequenceManifest manifest, CancellationToken cancellationToken) + { + var captured = CaptureTime(first); + var wanted = preferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, manifest.Title)); + if (wanted.Length == 0) wanted = "Sequence"; + var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + .Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase); + return ClaimFolder(paths, wanted, taken, manifest); + } + /// The first free variant of , claimed by creating its sidecar (first writer wins). private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet taken, SequenceManifest manifest) { @@ -197,24 +289,36 @@ public sealed class ArchiveIngest : IArchiveIngest { var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); var before = manifest.Frames.Count; - var known = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var f in manifest.Frames) - { - if (f.Sha256 is not null) known.TryAdd(f.Sha256, f); - } + var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase); + // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame. + var filed = new Dictionary(StringComparer.OrdinalIgnoreCase); var accepted = new List<(Pending Pending, SidecarFrame Frame)>(); + var repairs = new List<(Pending Pending, SidecarFrame Frame)>(); var duplicates = new List<(Pending Pending, Guid FrameId)>(); foreach (var p in batch) { - if (known.TryGetValue(p.Frame.Sha256, out var existing)) + if (filed.TryGetValue(p.Frame.Sha256, out var same)) { - duplicates.Add((p, existing.Id)); + duplicates.Add((p, same)); + continue; + } + // Only a frame that still holds the content makes this copy a duplicate; a missing or damaged one is repaired with it. + var holders = held[p.Frame.Sha256]; + if (holders.FirstOrDefault(f => IsIntact(sequence, f)) is { } intact) + { + duplicates.Add((p, intact.Id)); + continue; + } + if (holders.FirstOrDefault() is { } damaged) + { + filed[p.Frame.Sha256] = damaged.Id; + repairs.Add((p, damaged)); continue; } var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size, p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt); - known[p.Frame.Sha256] = frame; + filed[p.Frame.Sha256] = frame.Id; accepted.Add((p, frame)); } @@ -235,17 +339,37 @@ public sealed class ArchiveIngest : IArchiveIngest p.Completion.TrySetException(ex); } } + var repaired = new HashSet(); + foreach (var (p, frame) in repairs) + { + if (Repair(sequence, frame, p.Frame.StagedPath) is { } error) p.Completion.TrySetException(error); + else repaired.Add(frame.Id); + } var updated = SequenceFrames.Insert(manifest, moved); - if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated); + if (repaired.Count > 0) + { + updated = SequenceFrames.Update(updated, repaired.ToDictionary(id => id, + _ => (Func)(f => f with { Flags = f.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt) }))); + } + if (moved.Count > 0 || repaired.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated); IngestJournal.Delete(sequence); var byId = updated.Frames.ToDictionary(f => f.Id); - if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false); + if (moved.Count > 0 || repaired.Count > 0) + { + await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken) + .ConfigureAwait(false); + } foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); + foreach (var (p, frame) in repairs.Where(r => repaired.Contains(r.Frame.Id))) + p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); + var unrepaired = repairs.Select(r => r.Frame.Id).Where(id => !repaired.Contains(id)).ToHashSet(); foreach (var (p, frameId) in duplicates) { - if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true)); + // A copy of content this batch failed to file has nothing intact to stand for it. + if (byId.TryGetValue(frameId, out var original) && !unrepaired.Contains(frameId)) + p.Completion.TrySetResult(Result(sequenceId, sequence, original, true)); else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName))); } } @@ -284,11 +408,60 @@ public sealed class ArchiveIngest : IArchiveIngest return named; } - /// Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence. - private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, - CancellationToken cancellationToken) + /// The frame still holds its content: not flagged missing or corrupt, and its file is there with its size. + private static bool IsIntact(SequencePaths sequence, SidecarFrame frame) { - var inStep = appended.All(f => f.I >= before) + if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return false; + var file = new FileInfo(sequence.File(frame.File)); + return file.Exists && file.Length == frame.Size; + } + + /// + /// Puts a verified copy where a damaged frame's file belongs. A file still there (corrupt, or another size) moves aside + /// as name.damaged-<time>, which no scan takes for a photo, and is never deleted (BRIEF §0.2 rule 8); it comes + /// back when the copy cannot be moved in. Returns the failure, or null. + /// + private static Exception? Repair(SequencePaths sequence, SidecarFrame frame, string stagedPath) + { + var target = sequence.File(frame.File); + string? aside = null; + try + { + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + if (File.Exists(target)) + { + aside = string.Create(CultureInfo.InvariantCulture, $"{target}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}"); + File.Move(target, aside, overwrite: false); + } + File.Move(stagedPath, target, overwrite: false); + Log.Info("archive", $"Repaired {sequence.ArchiveRelative(frame.File)} with a verified copy"); + return null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + if (aside is not null && !File.Exists(target) && File.Exists(aside)) + { + try + { + File.Move(aside, target); + } + catch (Exception again) when (again is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"{aside} stays beside {target}", again); + } + } + return ex; + } + } + + /// + /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose + /// flags change) resyncs the whole sequence. + /// + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, + bool repaired, CancellationToken cancellationToken) + { + var inStep = !repaired && appended.All(f => f.I >= before) && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; if (!inStep) { diff --git a/src/Titano.Data/Archiving/IArchiveIngest.cs b/src/Titano.Data/Archiving/IArchiveIngest.cs index c8c1cb76..fa76d81d 100644 --- a/src/Titano.Data/Archiving/IArchiveIngest.cs +++ b/src/Titano.Data/Archiving/IArchiveIngest.cs @@ -17,7 +17,10 @@ public sealed record IngestRequest( DateTime? ClientCapturedAt, Guid? UserId); -/// The committed frame; when the sequence already held a frame with that SHA-256. +/// +/// The committed frame; when an intact frame of the sequence already held that SHA-256 (the file +/// stays in staging). A frame with that SHA-256 that was missing or damaged is repaired with the file instead (not a duplicate). +/// public sealed record IngestResult(Guid SequenceId, Guid FrameId, int Index, string RelativePath, bool Duplicate); /// diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index cb10e5e7..7cba5c45 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -62,10 +62,18 @@ public sealed class FrameRepository _db.ListAsync($"SELECT {Columns} FROM frames WHERE sequence_id = $1 AND idx > $2 ORDER BY idx LIMIT $3", [sequenceId, afterIndex, Cursor.ClampLimit(limit)], Map, cancellationToken); - /// Every frame with this content hash (duplicate detection, upload short-circuit). + /// Every frame with this content hash (duplicate detection). public Task> FindBySha256Async(string sha256, CancellationToken cancellationToken = default) => _db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant()], Map, cancellationToken); + /// + /// Frames with this content hash that are not flagged missing or corrupt: only such a copy may stand for an upload + /// (the upload short-circuit, a commit settled after a crash), so an upload of the same content repairs a damaged one. + /// + public Task> FindIntactBySha256Async(string sha256, CancellationToken cancellationToken = default) => + _db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 AND (flags & $2) = 0 ORDER BY sequence_id, idx", + [sha256.ToLowerInvariant(), (int)(FrameFlags.Missing | FrameFlags.Corrupt)], Map, cancellationToken); + /// Writes analysis results for many frames in one transaction; returns the number of frames updated. public Task UpdateLightStatsAsync(IReadOnlyList stats, CancellationToken cancellationToken = default) => _db.InTransactionAsync(async session => diff --git a/src/Titano.Server/Storage/UploadMaintenance.cs b/src/Titano.Server/Storage/UploadMaintenance.cs index 3abe04c1..2e53f72c 100644 --- a/src/Titano.Server/Storage/UploadMaintenance.cs +++ b/src/Titano.Server/Storage/UploadMaintenance.cs @@ -8,7 +8,8 @@ namespace Titano.Server.Storage; /// Upload housekeeping (BRIEF §14.4): at start, uploads a crash left half-committed are settled; then every half hour /// abandoned uploads expire, orphaned partial and staging files are removed, old upload records and dead tokens purged. /// -public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, UploadStore store, TimeProvider time, ILogger logger) +public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, ServerArchive archive, UploadStore store, TimeProvider time, + ILogger logger) : BackgroundService { private static readonly TimeSpan Interval = TimeSpan.FromMinutes(30); @@ -44,14 +45,16 @@ public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, } /// - /// Uploads left committing by a crash: committed when the frame with their hash exists, otherwise back to receiving - /// (the partial file is intact, so the client simply completes again). + /// Uploads left committing by a crash: committed when an intact frame with their hash exists, otherwise back to + /// receiving (the partial file is intact, so the client simply completes again and the ingest files or repairs it). /// public async Task RecoverAsync(CancellationToken cancellationToken) { foreach (var upload in await db.Uploads.ListByStateAsync(UploadState.Committing, cancellationToken)) { - var frame = upload.JobId is null ? (await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault() : null; + var frame = upload.JobId is null + ? (await db.Frames.FindIntactBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault(f => UploadRules.IsOnDisk(archive.Root, f)) + : null; if (frame is not null) { await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, frame.Id, cancellationToken); diff --git a/src/Titano.Server/Storage/UploadRules.cs b/src/Titano.Server/Storage/UploadRules.cs index 9dd31eef..db3fccd9 100644 --- a/src/Titano.Server/Storage/UploadRules.cs +++ b/src/Titano.Server/Storage/UploadRules.cs @@ -1,3 +1,5 @@ +using Titano.Core.Model; + namespace Titano.Server.Storage; /// Validation shared by frame and render uploads. @@ -30,6 +32,14 @@ public static class UploadRules return received; } + /// + /// The frame's original is in the archive with its recorded size. Together with flags that say neither missing nor + /// corrupt (FindIntactBySha256Async), that is the copy an upload of the same content may stand for; a damaged + /// one is repaired by uploading the content again. + /// + public static bool IsOnDisk(string archiveRoot, Frame frame) => + ArchivePathGuard.ExistingFile(archiveRoot, frame.RelativePath) is { } file && file.Length == frame.Size; + /// Free bytes on the volume holding , or null when the platform cannot tell. public static long? FreeBytes(string path) { diff --git a/src/Titano.Server/Storage/UploadService.cs b/src/Titano.Server/Storage/UploadService.cs index 0556474f..ebc57e30 100644 --- a/src/Titano.Server/Storage/UploadService.cs +++ b/src/Titano.Server/Storage/UploadService.cs @@ -16,9 +16,10 @@ using UploadStateDto = Titano.Protocol.UploadState; namespace Titano.Server.Storage; /// -/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256, resume an -/// unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the file on the archive -/// volume, verifies its hash and hands it to . +/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an +/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the +/// file on the archive volume, verifies its hash and hands it to , which repairs a missing or +/// damaged frame of that content. /// public sealed class UploadService( TitanoDatabase db, @@ -48,7 +49,8 @@ public sealed class UploadService( return Fail(404, "not_found", "sequence"); var sha = request.Sha256.ToLowerInvariant(); - var known = await db.Frames.FindBySha256Async(sha, cancellationToken); + // Only a copy that is still whole stands for the upload: a client may delete its file on this answer. + var known = (await db.Frames.FindIntactBySha256Async(sha, cancellationToken)).Where(f => UploadRules.IsOnDisk(archive.Root, f)).ToList(); if (known.Count > 0) { var frame = known.FirstOrDefault(f => f.SequenceId == request.SequenceId) ?? known[0]; diff --git a/src/Titano.Uploader/Uploads/UploadEngine.cs b/src/Titano.Uploader/Uploads/UploadEngine.cs index a6f26697..b37cf888 100644 --- a/src/Titano.Uploader/Uploads/UploadEngine.cs +++ b/src/Titano.Uploader/Uploads/UploadEngine.cs @@ -40,7 +40,8 @@ public interface IUploadEngine : INotifyPropertyChanged /// /// Works through the with concurrent files: hash, upload -/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source the server confirmed. +/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source whose bytes the server +/// received and verified. A file the server already held is kept: no copy of it was verified on this run. /// Lives on the UI thread: state changes happen there while hashing and HTTP run on the thread pool. /// public sealed class UploadEngine : IUploadEngine, IDisposable @@ -221,7 +222,7 @@ public sealed class UploadEngine : IUploadEngine, IDisposable item.State = outcome == UploadOutcome.Verified ? UploadItemState.Verified : UploadItemState.Skipped; _queue.OnChanged(); ItemConfirmed?.Invoke(this, item); - if (_deleteAfterUpload()) DeleteConfirmedSource(item); + if (outcome == UploadOutcome.Verified && _deleteAfterUpload()) DeleteConfirmedSource(item); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { @@ -312,8 +313,8 @@ public sealed class UploadEngine : IUploadEngine, IDisposable } /// - /// BRIEF §0.2 rule 8: deletes only a file the server confirmed by hash, and only when it is still exactly the - /// bytes that were hashed (same size and write time). + /// BRIEF §0.2 rule 8: deletes only a file whose bytes the server received and verified by hash before committing + /// them, and only when it is still exactly the bytes that were hashed (same size and write time). /// private static void DeleteConfirmedSource(UploadItem item) { diff --git a/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs b/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs index f2d940bb..0c6062fb 100644 --- a/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs +++ b/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs @@ -33,7 +33,7 @@ public sealed class UploadEngineTests(TestContext context) }); [Test] - public void DeleteAfterUploadRemovesOnlyConfirmedUnchangedFiles() => SingleThreadedContext.Run(async () => + public void DeleteAfterUploadRemovesOnlyVerifiedUnchangedFiles() => SingleThreadedContext.Run(async () => { var queue = Queue(UploadTargetMode.Sessions, 3); var (verified, existing, changed) = (queue.Items[0], queue.Items[1], queue.Items[2]); @@ -54,7 +54,8 @@ public sealed class UploadEngineTests(TestContext context) Assert.Equal(UploadItemState.Skipped, existing.State); Assert.False(File.Exists(verified.Path)); - Assert.False(File.Exists(existing.Path)); + // The server matched its hash without receiving these bytes: no verified copy, so the source stays. + Assert.True(File.Exists(existing.Path), "a file the server already held is kept"); Assert.True(File.Exists(changed.Path), "a file changed after verification is kept"); }); diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs index d9184407..30ad8460 100644 --- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -1,4 +1,5 @@ using Titano.Core.Archiving; +using Titano.Core.Model; using Titano.Data.Archiving; using Titano.Testing; using static Titano.Data.Tests.Archiving.ArchiveFixture; @@ -121,4 +122,97 @@ public sealed class ArchiveIngestTests(TestContext context) Assert.FileMissing(sequence.IngestJournalPath); Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count); } + + [Test] + public async Task AMissingFrameIsRepairedByTheSameContentInsteadOfCountingAsDuplicate() + { + await using var fx = await CreateAsync(context); + var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(original.SequenceId))!; + var file = Path.Combine(fx.Archive.RootPath, original.RelativePath); + var bytes = await File.ReadAllBytesAsync(file, fx.Ct); + File.Delete(file); + await fx.Service.Indexer.ReconcileAsync(fx.Archive, row, fx.Ct); + Assert.Equal(FrameFlags.Missing, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags); + + var again = fx.Stage(Start, 7); + var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct); + + Assert.False(repaired.Duplicate, "the server holds no intact copy: this one is filed"); + Assert.Equal((original.FrameId, original.Index, original.RelativePath), (repaired.FrameId, repaired.Index, repaired.RelativePath)); + Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct)); + Assert.FileMissing(again.Path); + Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath); + Assert.Equal(FrameFlags.None, manifest.Frames.Single().Flags); + Assert.False((await fx.Db.Sequences.GetAsync(original.SequenceId))!.Status.HasFlag(SequenceStatus.HasErrors)); + + // Intact again: the next copy is a duplicate and stays in staging. + var third = fx.Stage(Start, 7); + Assert.True((await fx.Service.Ingest.CommitAsync(Request(fx, third), fx.Ct)).Duplicate); + Assert.FileExists(third.Path); + } + + [Test] + public async Task ACorruptFrameIsRepairedAndItsDamagedFileKeptAside() + { + await using var fx = await CreateAsync(context); + var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct); + var file = Path.Combine(fx.Archive.RootPath, original.RelativePath); + var bytes = await File.ReadAllBytesAsync(file, fx.Ct); + var damaged = bytes.ToArray(); + damaged[^10] ^= 0xFF; + await File.WriteAllBytesAsync(file, damaged, fx.Ct); + await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal(FrameFlags.Corrupt, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags); + + var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct); + + Assert.False(repaired.Duplicate); + Assert.Equal(original.FrameId, repaired.FrameId); + Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct)); + var aside = Directory.GetFiles(Path.GetDirectoryName(file)!, Path.GetFileName(file) + ".damaged-*"); + Assert.Count(1, aside); + Assert.SequenceEqual(damaged, await File.ReadAllBytesAsync(aside[0], fx.Ct)); + Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags); + // The kept file is no photo to a scan: the sequence still has one frame. + Assert.Equal(0, (await fx.ScanAsync(ScanMode.Full)).Failed.Count); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount); + } + + [Test] + public async Task ASequenceCreatedWithoutPhotosGetsItsFolderWithTheFirstFrame() + { + await using var fx = await CreateAsync(context); + // As POST /api/v1/sequences creates it: a row without a folder. + var created = new Sequence + { + Id = Guid.CreateVersion7(), + ArchiveId = fx.Archive.Id, + Title = "Harbour sunset", + Folder = string.Empty, + Tags = ["harbour"], + Rating = 4, + Origin = SequenceOrigin.Server, + OwnerUserId = Guid.CreateVersion7(), + UpdatedAt = DateTimeOffset.UtcNow, + }; + await fx.Db.Sequences.UpsertAsync(created, fx.Ct); + + var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: created.Id), fx.Ct); + var second = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), sequenceId: created.Id, name: "IMG_0002.JPG"), fx.Ct); + + Assert.Equal((created.Id, created.Id), (first.SequenceId, second.SequenceId)); + var row = (await fx.Db.Sequences.GetAsync(created.Id))!; + Assert.Equal("2026/2026-09-28 Canon EOS R5", row.Folder); + Assert.Equal(("Harbour sunset", 4, SequenceOrigin.Server, created.OwnerUserId, 2), (row.Title, row.Rating, row.Origin, row.OwnerUserId, row.FrameCount)); + Assert.SequenceEqual(["harbour"], row.Tags); + Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", first.RelativePath); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath); + Assert.Equal((created.Id, "Harbour sunset", created.OwnerUserId.ToString()), (manifest.Id, manifest.Title, manifest.CreatedBy)); + Assert.SequenceEqual([first.FrameId, second.FrameId], manifest.Frames.Select(f => f.Id)); + Assert.FileMissing(fx.Paths.Sequence(string.Empty).ManifestPath); + // The index can be rebuilt from the sidecar it now has. + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } } diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs index 172540f8..0d58714b 100644 --- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs +++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs @@ -1,6 +1,8 @@ using System.Net; using Titano.Core.Archiving; using Titano.Core.Codecs.Jpeg; +using Titano.Protocol; +using Titano.Protocol.Client; using Titano.Server.Storage; using Titano.Testing; @@ -42,11 +44,69 @@ public sealed class ArchiveModuleTests(TestContext context) } } - private static byte[] Photo() + [Test] + public async Task ASequenceCreatedThroughTheApiTakesUploadsIntoAFolderOfItsOwn() + { + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register); + var (_, _, token) = await server.UserAsync("alberto", "editor"); + var source = context.CreateTempDirectory("source"); + var photos = new List<(string Path, byte[] Bytes)>(); + for (var i = 0; i < 2; i++) + { + var path = Path.Combine(source, $"IMG_000{i + 1}.JPG"); + var bytes = Photo(i + 1); + await File.WriteAllBytesAsync(path, bytes); + photos.Add((path, bytes)); + } + + // The Uploader's "New sequence" target: create it, then upload into it (BRIEF §15). + using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "Titano Uploader (tests)") { Token = token }; + var created = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null), context.Cancellation); + var first = Assert.NotNull(await client.UploadFileAsync(photos[0].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 11), ct: context.Cancellation)); + var second = Assert.NotNull(await client.UploadFileAsync(photos[1].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 21), ct: context.Cancellation)); + + Assert.Equal((created.Id, created.Id, 0, 1), (first.SequenceId, second.SequenceId, first.Index, second.Index)); + var row = Assert.NotNull(await server.Database.Sequences.GetAsync(created.Id)); + Assert.True(row.Folder.StartsWith("2026/2026-09-28", StringComparison.Ordinal), row.Folder); + Assert.Equal(("Harbour sunset", 2), (row.Title, row.FrameCount)); + var sidecar = ManifestStore.ReadSequence(new ArchivePaths(server.Options.ArchivePath).Sequence(row.Folder).ManifestPath); + Assert.Equal((created.Id, "Harbour sunset"), (sidecar.Id, sidecar.Title)); + Assert.SequenceEqual([first.FrameId, second.FrameId], sidecar.Frames.Select(f => f.Id)); + Assert.FileMissing(Path.Combine(server.Options.ArchivePath, ".titano", "sequence.json")); + for (var i = 0; i < 2; i++) + { + using var downloaded = new MemoryStream(); + await client.DownloadFrameAsync(i == 0 ? first.FrameId : second.FrameId, "original", downloaded, ct: context.Cancellation); + Assert.BytesEqual(photos[i].Bytes, downloaded.ToArray()); + } + } + + [Test] + public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace() + { + await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var photo = Photo(3); + var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof"); + var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId)); + var path = Path.Combine(server.Options.ArchivePath, row.RelativePath); + var known = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof", expected: HttpStatusCode.OK); + Assert.Equal((true, frame.FrameId), (known.AlreadyExists, known.FrameId)); + + File.Delete(path); + // No "already exists" for a copy that is gone: the bytes come again and the frame gets its file back. + var again = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof"); + + Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index)); + Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path)); + Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount); + } + + private static byte[] Photo(int seed = 0) { const int width = 96, height = 64; var pixels = new byte[width * height * 3]; - for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251); + for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251); return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 }); } } diff --git a/tests/Titano.Server.Tests/Storage/UploadTests.cs b/tests/Titano.Server.Tests/Storage/UploadTests.cs index ae889782..aea2324e 100644 --- a/tests/Titano.Server.Tests/Storage/UploadTests.cs +++ b/tests/Titano.Server.Tests/Storage/UploadTests.cs @@ -1,6 +1,7 @@ using System.Net; using System.Net.Http.Json; using Microsoft.Extensions.DependencyInjection; +using Titano.Core.Model; using Titano.Protocol; using Titano.Protocol.Client; using Titano.Server.Storage; @@ -96,6 +97,49 @@ public sealed class UploadTests(TestContext context) Assert.Equal(frame.FrameId, again.FrameId); } + [Test] + public async Task ACopyFlaggedOrGoneIsNoShortcut() + { + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk); + var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader"); + // A client may delete its file on "already exists": a damaged copy must ask for the bytes again. + var flagged = Uploads.Photo(5000, 11); + var first = await Uploads.UploadAsync(uploader, flagged); + await server.Database.Frames.UpdateFlagsAsync([first.FrameId], FrameFlags.Corrupt, FrameFlags.None); + Assert.False((await Uploads.StartAsync(uploader, flagged)).AlreadyExists); + + var gone = Uploads.Photo(5000, 12); + var second = await Uploads.UploadAsync(uploader, gone); + File.Delete(Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(second.FrameId)).RelativePath)); + Assert.False((await Uploads.StartAsync(uploader, gone)).AlreadyExists); + } + + [Test] + public async Task ACommitLeftByACrashIsSettledOnlyByAnIntactFrame() + { + await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk); + var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader"); + var data = Uploads.Photo(Chunk + 100, 13); + var missing = await Uploads.UploadAsync(uploader, data); + await server.Database.Frames.UpdateFlagsAsync([missing.FrameId], FrameFlags.Missing, FrameFlags.None); + var start = await Uploads.StartAsync(uploader, data); + var id = Assert.NotNull(start.UploadId); + for (var i = 0; i < 2; i++) + { + using var put = await Uploads.PutChunkAsync(uploader, id, i, Uploads.Chunk(data, Chunk, i)); + await put.ExpectAsync(HttpStatusCode.OK); + } + // The server stopped while committing it. + Assert.True(await server.Database.Uploads.TryTransitionAsync(id, Titano.Data.Repositories.UploadState.Receiving, + Titano.Data.Repositories.UploadState.Committing, cancellationToken: context.Cancellation)); + + await server.Services.GetRequiredService().RecoverAsync(context.Cancellation); + + // The flagged frame does not stand for it: the client completes again with its part file intact. + using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State); + Assert.True(server.Services.GetRequiredService().Exists(id)); + } + [Test] public async Task AnUploadKeepsItsChunkSizeWhenTheSettingShrinks() {