diff --git a/src/Titano.App/Import/ServerImportSink.cs b/src/Titano.App/Import/ServerImportSink.cs
index d65b352e..b0feb616 100644
--- a/src/Titano.App/Import/ServerImportSink.cs
+++ b/src/Titano.App/Import/ServerImportSink.cs
@@ -14,7 +14,8 @@ namespace Titano.App.Import;
/// server files them in one sequence — the same one when an interrupted import is started again. A typed session
/// title is set on that sequence once. A file the server refuses (a hash mismatch, a format it rejects) is listed in
/// and the import goes on (BRIEF §13: errors per file); a failed connection, sign-in or server
-/// stops it.
+/// stops it. A file the server already held counts as a duplicate: the server answers so only for a copy that is not
+/// flagged missing or damaged, and a server import never deletes anything from the card.
///
public sealed class ServerImportSink(TitanoClient client, string stagingDirectory) : IImportSink
{
diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs
index 29a31c2a..c23f1b86 100644
--- a/src/Titano.Data/Archiving/ArchiveIngest.cs
+++ b/src/Titano.Data/Archiving/ArchiveIngest.cs
@@ -1,6 +1,8 @@
using System.Collections.Concurrent;
using System.Globalization;
using Titano.Core.Archiving;
+using Titano.Core.Diagnostics;
+using Titano.Core.IO;
using Titano.Core.Localization;
using Titano.Core.Metadata;
using Titano.Core.Model;
@@ -22,7 +24,9 @@ internal sealed record SequenceTarget(string Key, string? Title, string? Preferr
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
-/// SHA-256 the sequence already holds is reported as a duplicate and left in staging.
+/// SHA-256 an intact frame of the sequence already holds is reported as a duplicate and left in staging; a frame with
+/// that SHA-256 that is flagged missing or corrupt, or whose file is gone or has another size, is repaired with it
+/// instead (its damaged file kept beside it). A sequence created without photos gets its folder with its first frame.
///
public sealed class ArchiveIngest : IArchiveIngest
{
@@ -57,6 +61,7 @@ public sealed class ArchiveIngest : IArchiveIngest
{
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
+ if (AwaitsFolder(sequence, new ArchivePaths(archive.RootPath))) await PlaceAsync(archive, id, frame, cancellationToken).ConfigureAwait(false);
sequenceId = id;
}
else
@@ -110,12 +115,7 @@ public sealed class ArchiveIngest : IArchiveIngest
Created = now,
UpdatedAt = now,
};
- var captured = CaptureTime(first);
- var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title));
- if (wanted.Length == 0) wanted = "Sequence";
- var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
- .Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
- var sequence = ClaimFolder(paths, wanted, taken, manifest);
+ var sequence = await ClaimNewFolderAsync(archive, paths, target.PreferredFolder, first, manifest, cancellationToken).ConfigureAwait(false);
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
@@ -134,6 +134,98 @@ public sealed class ArchiveIngest : IArchiveIngest
return manifest.Id;
}
+ ///
+ /// A sequence created without photos (POST /api/v1/sequences, BRIEF §14.3) has no folder until its first frame.
+ /// An empty folder otherwise means the archive root itself, adopted in place, whose sidecar names the sequence.
+ ///
+ internal static bool AwaitsFolder(Sequence sequence, ArchivePaths paths)
+ {
+ if (sequence.Folder.Length > 0) return false;
+ var root = paths.Sequence(string.Empty);
+ if (!File.Exists(root.ManifestPath)) return true;
+ try
+ {
+ return ManifestStore.ReadSequence(root.ManifestPath).Id != sequence.Id;
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException)
+ {
+ // An unreadable root sidecar: the commit reports it rather than placing the sequence elsewhere.
+ return false;
+ }
+ }
+
+ ///
+ /// Gives a sequence created without photos its folder, named from its first frame and title by the same rules as a
+ /// sequence created by an upload, with a sidecar carrying the row's id, title, tags, rating and notes (and its project
+ /// when one was saved); the index row takes the folder in one transaction. Serialised with the creation of sequences.
+ ///
+ private async Task PlaceAsync(Archive archive, Guid sequenceId, StagedFrame first, CancellationToken cancellationToken)
+ {
+ await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ var paths = new ArchivePaths(archive.RootPath);
+ var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
+ ?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
+ // A concurrent first frame placed it meanwhile.
+ if (!AwaitsFolder(row, paths)) return;
+ ArchiveVolume.EnsureWritable(paths);
+ var now = DateTimeOffset.UtcNow;
+ var m = first.Metadata;
+ var manifest = new SequenceManifest
+ {
+ Id = row.Id,
+ Title = row.Title,
+ // The photo names its camera; the creator's hint stands in when it does not.
+ Camera = m.Make is null && m.Model is null && row.CameraModel is not null
+ ? new SidecarCamera { Model = row.CameraModel }
+ : new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
+ Lens = m.LensModel,
+ Tags = row.Tags,
+ Rating = row.Rating,
+ Notes = row.Notes,
+ Versions = new ManifestVersions { Settings = row.SettingsVersion, Frames = row.FramesVersion },
+ CreatedBy = row.OwnerUserId?.ToString(),
+ Created = now,
+ UpdatedAt = now,
+ };
+ var sequence = await ClaimNewFolderAsync(archive, paths, null, first, manifest, cancellationToken).ConfigureAwait(false);
+ try
+ {
+ if (row.SettingsJson is { } settings) AtomicFile.WriteAllText(sequence.ProjectPath, settings);
+ await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
+ var current = await tx.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
+ ?? throw new InvalidOperationException($"Sequence {sequenceId} was deleted before its first frame.");
+ await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, current), cancellationToken).ConfigureAwait(false);
+ await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
+ }
+ catch
+ {
+ // Nothing was filed into the folder yet: it goes, and the next first frame claims one again.
+ if (File.Exists(sequence.ProjectPath)) File.Delete(sequence.ProjectPath);
+ ReleaseFolder(sequence);
+ throw;
+ }
+ await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
+ }
+ finally
+ {
+ _creation.Release();
+ }
+ }
+
+ /// A new folder for : the plan's folder or the archive's folder template, made unique.
+ private async Task ClaimNewFolderAsync(Archive archive, ArchivePaths paths, string? preferredFolder, StagedFrame first,
+ SequenceManifest manifest, CancellationToken cancellationToken)
+ {
+ var captured = CaptureTime(first);
+ var wanted = preferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, manifest.Title));
+ if (wanted.Length == 0) wanted = "Sequence";
+ var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
+ .Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
+ return ClaimFolder(paths, wanted, taken, manifest);
+ }
+
/// The first free variant of , claimed by creating its sidecar (first writer wins).
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet taken, SequenceManifest manifest)
{
@@ -197,24 +289,36 @@ public sealed class ArchiveIngest : IArchiveIngest
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
var before = manifest.Frames.Count;
- var known = new Dictionary(StringComparer.OrdinalIgnoreCase);
- foreach (var f in manifest.Frames)
- {
- if (f.Sha256 is not null) known.TryAdd(f.Sha256, f);
- }
+ var held = manifest.Frames.Where(f => f.Sha256 is not null).ToLookup(f => f.Sha256!, StringComparer.OrdinalIgnoreCase);
+ // Content this batch files (new frames and repairs): another copy of it in the batch is a duplicate of that frame.
+ var filed = new Dictionary(StringComparer.OrdinalIgnoreCase);
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
+ var repairs = new List<(Pending Pending, SidecarFrame Frame)>();
var duplicates = new List<(Pending Pending, Guid FrameId)>();
foreach (var p in batch)
{
- if (known.TryGetValue(p.Frame.Sha256, out var existing))
+ if (filed.TryGetValue(p.Frame.Sha256, out var same))
{
- duplicates.Add((p, existing.Id));
+ duplicates.Add((p, same));
+ continue;
+ }
+ // Only a frame that still holds the content makes this copy a duplicate; a missing or damaged one is repaired with it.
+ var holders = held[p.Frame.Sha256];
+ if (holders.FirstOrDefault(f => IsIntact(sequence, f)) is { } intact)
+ {
+ duplicates.Add((p, intact.Id));
+ continue;
+ }
+ if (holders.FirstOrDefault() is { } damaged)
+ {
+ filed[p.Frame.Sha256] = damaged.Id;
+ repairs.Add((p, damaged));
continue;
}
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
- known[p.Frame.Sha256] = frame;
+ filed[p.Frame.Sha256] = frame.Id;
accepted.Add((p, frame));
}
@@ -235,17 +339,37 @@ public sealed class ArchiveIngest : IArchiveIngest
p.Completion.TrySetException(ex);
}
}
+ var repaired = new HashSet();
+ foreach (var (p, frame) in repairs)
+ {
+ if (Repair(sequence, frame, p.Frame.StagedPath) is { } error) p.Completion.TrySetException(error);
+ else repaired.Add(frame.Id);
+ }
var updated = SequenceFrames.Insert(manifest, moved);
- if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
+ if (repaired.Count > 0)
+ {
+ updated = SequenceFrames.Update(updated, repaired.ToDictionary(id => id,
+ _ => (Func)(f => f with { Flags = f.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt) })));
+ }
+ if (moved.Count > 0 || repaired.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
IngestJournal.Delete(sequence);
var byId = updated.Frames.ToDictionary(f => f.Id);
- if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false);
+ if (moved.Count > 0 || repaired.Count > 0)
+ {
+ await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), repaired.Count > 0, cancellationToken)
+ .ConfigureAwait(false);
+ }
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
+ foreach (var (p, frame) in repairs.Where(r => repaired.Contains(r.Frame.Id)))
+ p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
+ var unrepaired = repairs.Select(r => r.Frame.Id).Where(id => !repaired.Contains(id)).ToHashSet();
foreach (var (p, frameId) in duplicates)
{
- if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
+ // A copy of content this batch failed to file has nothing intact to stand for it.
+ if (byId.TryGetValue(frameId, out var original) && !unrepaired.Contains(frameId))
+ p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
}
}
@@ -284,11 +408,60 @@ public sealed class ArchiveIngest : IArchiveIngest
return named;
}
- /// Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence.
- private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended,
- CancellationToken cancellationToken)
+ /// The frame still holds its content: not flagged missing or corrupt, and its file is there with its size.
+ private static bool IsIntact(SequencePaths sequence, SidecarFrame frame)
{
- var inStep = appended.All(f => f.I >= before)
+ if ((frame.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0) return false;
+ var file = new FileInfo(sequence.File(frame.File));
+ return file.Exists && file.Length == frame.Size;
+ }
+
+ ///
+ /// Puts a verified copy where a damaged frame's file belongs. A file still there (corrupt, or another size) moves aside
+ /// as name.damaged-<time>, which no scan takes for a photo, and is never deleted (BRIEF §0.2 rule 8); it comes
+ /// back when the copy cannot be moved in. Returns the failure, or null.
+ ///
+ private static Exception? Repair(SequencePaths sequence, SidecarFrame frame, string stagedPath)
+ {
+ var target = sequence.File(frame.File);
+ string? aside = null;
+ try
+ {
+ Directory.CreateDirectory(Path.GetDirectoryName(target)!);
+ if (File.Exists(target))
+ {
+ aside = string.Create(CultureInfo.InvariantCulture, $"{target}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
+ File.Move(target, aside, overwrite: false);
+ }
+ File.Move(stagedPath, target, overwrite: false);
+ Log.Info("archive", $"Repaired {sequence.ArchiveRelative(frame.File)} with a verified copy");
+ return null;
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ if (aside is not null && !File.Exists(target) && File.Exists(aside))
+ {
+ try
+ {
+ File.Move(aside, target);
+ }
+ catch (Exception again) when (again is IOException or UnauthorizedAccessException)
+ {
+ Log.Warning("archive", $"{aside} stays beside {target}", again);
+ }
+ }
+ return ex;
+ }
+ }
+
+ ///
+ /// Appends the new rows when the index was in step with the sidecar; otherwise (or when frames were repaired, whose
+ /// flags change) resyncs the whole sequence.
+ ///
+ private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended,
+ bool repaired, CancellationToken cancellationToken)
+ {
+ var inStep = !repaired && appended.All(f => f.I >= before)
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
if (!inStep)
{
diff --git a/src/Titano.Data/Archiving/IArchiveIngest.cs b/src/Titano.Data/Archiving/IArchiveIngest.cs
index c8c1cb76..fa76d81d 100644
--- a/src/Titano.Data/Archiving/IArchiveIngest.cs
+++ b/src/Titano.Data/Archiving/IArchiveIngest.cs
@@ -17,7 +17,10 @@ public sealed record IngestRequest(
DateTime? ClientCapturedAt,
Guid? UserId);
-/// The committed frame; when the sequence already held a frame with that SHA-256.
+///
+/// The committed frame; when an intact frame of the sequence already held that SHA-256 (the file
+/// stays in staging). A frame with that SHA-256 that was missing or damaged is repaired with the file instead (not a duplicate).
+///
public sealed record IngestResult(Guid SequenceId, Guid FrameId, int Index, string RelativePath, bool Duplicate);
///
diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs
index cb10e5e7..7cba5c45 100644
--- a/src/Titano.Data/Repositories/FrameRepository.cs
+++ b/src/Titano.Data/Repositories/FrameRepository.cs
@@ -62,10 +62,18 @@ public sealed class FrameRepository
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sequence_id = $1 AND idx > $2 ORDER BY idx LIMIT $3",
[sequenceId, afterIndex, Cursor.ClampLimit(limit)], Map, cancellationToken);
- /// Every frame with this content hash (duplicate detection, upload short-circuit).
+ /// Every frame with this content hash (duplicate detection).
public Task> FindBySha256Async(string sha256, CancellationToken cancellationToken = default) =>
_db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 ORDER BY sequence_id, idx", [sha256.ToLowerInvariant()], Map, cancellationToken);
+ ///
+ /// Frames with this content hash that are not flagged missing or corrupt: only such a copy may stand for an upload
+ /// (the upload short-circuit, a commit settled after a crash), so an upload of the same content repairs a damaged one.
+ ///
+ public Task> FindIntactBySha256Async(string sha256, CancellationToken cancellationToken = default) =>
+ _db.ListAsync($"SELECT {Columns} FROM frames WHERE sha256 = $1 AND (flags & $2) = 0 ORDER BY sequence_id, idx",
+ [sha256.ToLowerInvariant(), (int)(FrameFlags.Missing | FrameFlags.Corrupt)], Map, cancellationToken);
+
/// Writes analysis results for many frames in one transaction; returns the number of frames updated.
public Task UpdateLightStatsAsync(IReadOnlyList stats, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
diff --git a/src/Titano.Server/Storage/UploadMaintenance.cs b/src/Titano.Server/Storage/UploadMaintenance.cs
index 3abe04c1..2e53f72c 100644
--- a/src/Titano.Server/Storage/UploadMaintenance.cs
+++ b/src/Titano.Server/Storage/UploadMaintenance.cs
@@ -8,7 +8,8 @@ namespace Titano.Server.Storage;
/// Upload housekeeping (BRIEF §14.4): at start, uploads a crash left half-committed are settled; then every half hour
/// abandoned uploads expire, orphaned partial and staging files are removed, old upload records and dead tokens purged.
///
-public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, UploadStore store, TimeProvider time, ILogger logger)
+public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options, ServerArchive archive, UploadStore store, TimeProvider time,
+ ILogger logger)
: BackgroundService
{
private static readonly TimeSpan Interval = TimeSpan.FromMinutes(30);
@@ -44,14 +45,16 @@ public sealed class UploadMaintenance(TitanoDatabase db, ServerOptions options,
}
///
- /// Uploads left committing by a crash: committed when the frame with their hash exists, otherwise back to receiving
- /// (the partial file is intact, so the client simply completes again).
+ /// Uploads left committing by a crash: committed when an intact frame with their hash exists, otherwise back to
+ /// receiving (the partial file is intact, so the client simply completes again and the ingest files or repairs it).
///
public async Task RecoverAsync(CancellationToken cancellationToken)
{
foreach (var upload in await db.Uploads.ListByStateAsync(UploadState.Committing, cancellationToken))
{
- var frame = upload.JobId is null ? (await db.Frames.FindBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault() : null;
+ var frame = upload.JobId is null
+ ? (await db.Frames.FindIntactBySha256Async(upload.Sha256, cancellationToken)).FirstOrDefault(f => UploadRules.IsOnDisk(archive.Root, f))
+ : null;
if (frame is not null)
{
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Committed, frame.Id, cancellationToken);
diff --git a/src/Titano.Server/Storage/UploadRules.cs b/src/Titano.Server/Storage/UploadRules.cs
index 9dd31eef..db3fccd9 100644
--- a/src/Titano.Server/Storage/UploadRules.cs
+++ b/src/Titano.Server/Storage/UploadRules.cs
@@ -1,3 +1,5 @@
+using Titano.Core.Model;
+
namespace Titano.Server.Storage;
/// Validation shared by frame and render uploads.
@@ -30,6 +32,14 @@ public static class UploadRules
return received;
}
+ ///
+ /// The frame's original is in the archive with its recorded size. Together with flags that say neither missing nor
+ /// corrupt (FindIntactBySha256Async), that is the copy an upload of the same content may stand for; a damaged
+ /// one is repaired by uploading the content again.
+ ///
+ public static bool IsOnDisk(string archiveRoot, Frame frame) =>
+ ArchivePathGuard.ExistingFile(archiveRoot, frame.RelativePath) is { } file && file.Length == frame.Size;
+
/// Free bytes on the volume holding , or null when the platform cannot tell.
public static long? FreeBytes(string path)
{
diff --git a/src/Titano.Server/Storage/UploadService.cs b/src/Titano.Server/Storage/UploadService.cs
index 0556474f..ebc57e30 100644
--- a/src/Titano.Server/Storage/UploadService.cs
+++ b/src/Titano.Server/Storage/UploadService.cs
@@ -16,9 +16,10 @@ using UploadStateDto = Titano.Protocol.UploadState;
namespace Titano.Server.Storage;
///
-/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256, resume an
-/// unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the file on the archive
-/// volume, verifies its hash and hands it to .
+/// Chunked, resumable, hash-verified frame uploads (BRIEF §14.4): start (short-circuit on a known SHA-256 held by an
+/// intact frame, resume an unfinished upload, quota and disk checks), idempotent chunks, and completion that stages the
+/// file on the archive volume, verifies its hash and hands it to , which repairs a missing or
+/// damaged frame of that content.
///
public sealed class UploadService(
TitanoDatabase db,
@@ -48,7 +49,8 @@ public sealed class UploadService(
return Fail(404, "not_found", "sequence");
var sha = request.Sha256.ToLowerInvariant();
- var known = await db.Frames.FindBySha256Async(sha, cancellationToken);
+ // Only a copy that is still whole stands for the upload: a client may delete its file on this answer.
+ var known = (await db.Frames.FindIntactBySha256Async(sha, cancellationToken)).Where(f => UploadRules.IsOnDisk(archive.Root, f)).ToList();
if (known.Count > 0)
{
var frame = known.FirstOrDefault(f => f.SequenceId == request.SequenceId) ?? known[0];
diff --git a/src/Titano.Uploader/Uploads/UploadEngine.cs b/src/Titano.Uploader/Uploads/UploadEngine.cs
index a6f26697..b37cf888 100644
--- a/src/Titano.Uploader/Uploads/UploadEngine.cs
+++ b/src/Titano.Uploader/Uploads/UploadEngine.cs
@@ -40,7 +40,8 @@ public interface IUploadEngine : INotifyPropertyChanged
///
/// Works through the with concurrent files: hash, upload
-/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source the server confirmed.
+/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source whose bytes the server
+/// received and verified. A file the server already held is kept: no copy of it was verified on this run.
/// Lives on the UI thread: state changes happen there while hashing and HTTP run on the thread pool.
///
public sealed class UploadEngine : IUploadEngine, IDisposable
@@ -221,7 +222,7 @@ public sealed class UploadEngine : IUploadEngine, IDisposable
item.State = outcome == UploadOutcome.Verified ? UploadItemState.Verified : UploadItemState.Skipped;
_queue.OnChanged();
ItemConfirmed?.Invoke(this, item);
- if (_deleteAfterUpload()) DeleteConfirmedSource(item);
+ if (outcome == UploadOutcome.Verified && _deleteAfterUpload()) DeleteConfirmedSource(item);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
@@ -312,8 +313,8 @@ public sealed class UploadEngine : IUploadEngine, IDisposable
}
///
- /// BRIEF §0.2 rule 8: deletes only a file the server confirmed by hash, and only when it is still exactly the
- /// bytes that were hashed (same size and write time).
+ /// BRIEF §0.2 rule 8: deletes only a file whose bytes the server received and verified by hash before committing
+ /// them, and only when it is still exactly the bytes that were hashed (same size and write time).
///
private static void DeleteConfirmedSource(UploadItem item)
{
diff --git a/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs b/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs
index f2d940bb..0c6062fb 100644
--- a/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs
+++ b/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs
@@ -33,7 +33,7 @@ public sealed class UploadEngineTests(TestContext context)
});
[Test]
- public void DeleteAfterUploadRemovesOnlyConfirmedUnchangedFiles() => SingleThreadedContext.Run(async () =>
+ public void DeleteAfterUploadRemovesOnlyVerifiedUnchangedFiles() => SingleThreadedContext.Run(async () =>
{
var queue = Queue(UploadTargetMode.Sessions, 3);
var (verified, existing, changed) = (queue.Items[0], queue.Items[1], queue.Items[2]);
@@ -54,7 +54,8 @@ public sealed class UploadEngineTests(TestContext context)
Assert.Equal(UploadItemState.Skipped, existing.State);
Assert.False(File.Exists(verified.Path));
- Assert.False(File.Exists(existing.Path));
+ // The server matched its hash without receiving these bytes: no verified copy, so the source stays.
+ Assert.True(File.Exists(existing.Path), "a file the server already held is kept");
Assert.True(File.Exists(changed.Path), "a file changed after verification is kept");
});
diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
index d9184407..30ad8460 100644
--- a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
+++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs
@@ -1,4 +1,5 @@
using Titano.Core.Archiving;
+using Titano.Core.Model;
using Titano.Data.Archiving;
using Titano.Testing;
using static Titano.Data.Tests.Archiving.ArchiveFixture;
@@ -121,4 +122,97 @@ public sealed class ArchiveIngestTests(TestContext context)
Assert.FileMissing(sequence.IngestJournalPath);
Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count);
}
+
+ [Test]
+ public async Task AMissingFrameIsRepairedByTheSameContentInsteadOfCountingAsDuplicate()
+ {
+ await using var fx = await CreateAsync(context);
+ var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
+ var row = (await fx.Db.Sequences.GetAsync(original.SequenceId))!;
+ var file = Path.Combine(fx.Archive.RootPath, original.RelativePath);
+ var bytes = await File.ReadAllBytesAsync(file, fx.Ct);
+ File.Delete(file);
+ await fx.Service.Indexer.ReconcileAsync(fx.Archive, row, fx.Ct);
+ Assert.Equal(FrameFlags.Missing, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
+
+ var again = fx.Stage(Start, 7);
+ var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct);
+
+ Assert.False(repaired.Duplicate, "the server holds no intact copy: this one is filed");
+ Assert.Equal((original.FrameId, original.Index, original.RelativePath), (repaired.FrameId, repaired.Index, repaired.RelativePath));
+ Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
+ Assert.FileMissing(again.Path);
+ Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
+ var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
+ Assert.Equal(FrameFlags.None, manifest.Frames.Single().Flags);
+ Assert.False((await fx.Db.Sequences.GetAsync(original.SequenceId))!.Status.HasFlag(SequenceStatus.HasErrors));
+
+ // Intact again: the next copy is a duplicate and stays in staging.
+ var third = fx.Stage(Start, 7);
+ Assert.True((await fx.Service.Ingest.CommitAsync(Request(fx, third), fx.Ct)).Duplicate);
+ Assert.FileExists(third.Path);
+ }
+
+ [Test]
+ public async Task ACorruptFrameIsRepairedAndItsDamagedFileKeptAside()
+ {
+ await using var fx = await CreateAsync(context);
+ var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
+ var file = Path.Combine(fx.Archive.RootPath, original.RelativePath);
+ var bytes = await File.ReadAllBytesAsync(file, fx.Ct);
+ var damaged = bytes.ToArray();
+ damaged[^10] ^= 0xFF;
+ await File.WriteAllBytesAsync(file, damaged, fx.Ct);
+ await fx.Service.Verifier.VerifyAsync(fx.Archive, null, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct);
+ Assert.Equal(FrameFlags.Corrupt, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
+
+ var repaired = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct);
+
+ Assert.False(repaired.Duplicate);
+ Assert.Equal(original.FrameId, repaired.FrameId);
+ Assert.SequenceEqual(bytes, await File.ReadAllBytesAsync(file, fx.Ct));
+ var aside = Directory.GetFiles(Path.GetDirectoryName(file)!, Path.GetFileName(file) + ".damaged-*");
+ Assert.Count(1, aside);
+ Assert.SequenceEqual(damaged, await File.ReadAllBytesAsync(aside[0], fx.Ct));
+ Assert.Equal(FrameFlags.None, (await fx.Db.Frames.GetAsync(original.FrameId))!.Flags);
+ // The kept file is no photo to a scan: the sequence still has one frame.
+ Assert.Equal(0, (await fx.ScanAsync(ScanMode.Full)).Failed.Count);
+ Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount);
+ }
+
+ [Test]
+ public async Task ASequenceCreatedWithoutPhotosGetsItsFolderWithTheFirstFrame()
+ {
+ await using var fx = await CreateAsync(context);
+ // As POST /api/v1/sequences creates it: a row without a folder.
+ var created = new Sequence
+ {
+ Id = Guid.CreateVersion7(),
+ ArchiveId = fx.Archive.Id,
+ Title = "Harbour sunset",
+ Folder = string.Empty,
+ Tags = ["harbour"],
+ Rating = 4,
+ Origin = SequenceOrigin.Server,
+ OwnerUserId = Guid.CreateVersion7(),
+ UpdatedAt = DateTimeOffset.UtcNow,
+ };
+ await fx.Db.Sequences.UpsertAsync(created, fx.Ct);
+
+ var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), sequenceId: created.Id), fx.Ct);
+ var second = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), sequenceId: created.Id, name: "IMG_0002.JPG"), fx.Ct);
+
+ Assert.Equal((created.Id, created.Id), (first.SequenceId, second.SequenceId));
+ var row = (await fx.Db.Sequences.GetAsync(created.Id))!;
+ Assert.Equal("2026/2026-09-28 Canon EOS R5", row.Folder);
+ Assert.Equal(("Harbour sunset", 4, SequenceOrigin.Server, created.OwnerUserId, 2), (row.Title, row.Rating, row.Origin, row.OwnerUserId, row.FrameCount));
+ Assert.SequenceEqual(["harbour"], row.Tags);
+ Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", first.RelativePath);
+ var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath);
+ Assert.Equal((created.Id, "Harbour sunset", created.OwnerUserId.ToString()), (manifest.Id, manifest.Title, manifest.CreatedBy));
+ Assert.SequenceEqual([first.FrameId, second.FrameId], manifest.Frames.Select(f => f.Id));
+ Assert.FileMissing(fx.Paths.Sequence(string.Empty).ManifestPath);
+ // The index can be rebuilt from the sidecar it now has.
+ Assert.Equal(0, (await fx.ScanAsync()).Indexed);
+ }
}
diff --git a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
index 172540f8..0d58714b 100644
--- a/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
+++ b/tests/Titano.Server.Tests/Storage/ArchiveModuleTests.cs
@@ -1,6 +1,8 @@
using System.Net;
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
+using Titano.Protocol;
+using Titano.Protocol.Client;
using Titano.Server.Storage;
using Titano.Testing;
@@ -42,11 +44,69 @@ public sealed class ArchiveModuleTests(TestContext context)
}
}
- private static byte[] Photo()
+ [Test]
+ public async Task ASequenceCreatedThroughTheApiTakesUploadsIntoAFolderOfItsOwn()
+ {
+ await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = 64 * 1024, ArchiveModule.Register);
+ var (_, _, token) = await server.UserAsync("alberto", "editor");
+ var source = context.CreateTempDirectory("source");
+ var photos = new List<(string Path, byte[] Bytes)>();
+ for (var i = 0; i < 2; i++)
+ {
+ var path = Path.Combine(source, $"IMG_000{i + 1}.JPG");
+ var bytes = Photo(i + 1);
+ await File.WriteAllBytesAsync(path, bytes);
+ photos.Add((path, bytes));
+ }
+
+ // The Uploader's "New sequence" target: create it, then upload into it (BRIEF §15).
+ using var client = new TitanoClient(server.HttpBase, new CertificatePinning([]), "Titano Uploader (tests)") { Token = token };
+ var created = await client.CreateSequenceAsync(new CreateSequenceRequest("Harbour sunset", null), context.Cancellation);
+ var first = Assert.NotNull(await client.UploadFileAsync(photos[0].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 11), ct: context.Cancellation));
+ var second = Assert.NotNull(await client.UploadFileAsync(photos[1].Path, created.Id, new DateTime(2026, 9, 28, 18, 2, 21), ct: context.Cancellation));
+
+ Assert.Equal((created.Id, created.Id, 0, 1), (first.SequenceId, second.SequenceId, first.Index, second.Index));
+ var row = Assert.NotNull(await server.Database.Sequences.GetAsync(created.Id));
+ Assert.True(row.Folder.StartsWith("2026/2026-09-28", StringComparison.Ordinal), row.Folder);
+ Assert.Equal(("Harbour sunset", 2), (row.Title, row.FrameCount));
+ var sidecar = ManifestStore.ReadSequence(new ArchivePaths(server.Options.ArchivePath).Sequence(row.Folder).ManifestPath);
+ Assert.Equal((created.Id, "Harbour sunset"), (sidecar.Id, sidecar.Title));
+ Assert.SequenceEqual([first.FrameId, second.FrameId], sidecar.Frames.Select(f => f.Id));
+ Assert.FileMissing(Path.Combine(server.Options.ArchivePath, ".titano", "sequence.json"));
+ for (var i = 0; i < 2; i++)
+ {
+ using var downloaded = new MemoryStream();
+ await client.DownloadFrameAsync(i == 0 ? first.FrameId : second.FrameId, "original", downloaded, ct: context.Cancellation);
+ Assert.BytesEqual(photos[i].Bytes, downloaded.ToArray());
+ }
+ }
+
+ [Test]
+ public async Task AnOriginalWhoseFileIsGoneIsUploadedAgainIntoItsPlace()
+ {
+ await using var server = await TestServer.StartAsync(context, services: ArchiveModule.Register);
+ var (editor, _, _) = await server.UserAsync("alberto", "editor");
+ var photo = Photo(3);
+ var frame = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
+ var row = Assert.NotNull(await server.Database.Frames.GetAsync(frame.FrameId));
+ var path = Path.Combine(server.Options.ArchivePath, row.RelativePath);
+ var known = await Uploads.StartAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof", expected: HttpStatusCode.OK);
+ Assert.Equal((true, frame.FrameId), (known.AlreadyExists, known.FrameId));
+
+ File.Delete(path);
+ // No "already exists" for a copy that is gone: the bytes come again and the frame gets its file back.
+ var again = await Uploads.UploadAsync(editor, photo, "IMG_0001.JPG", sessionKey: "roof");
+
+ Assert.Equal((frame.FrameId, frame.SequenceId, frame.Index), (again.FrameId, again.SequenceId, again.Index));
+ Assert.SequenceEqual(photo, await File.ReadAllBytesAsync(path));
+ Assert.Equal(1, Assert.NotNull(await server.Database.Sequences.GetAsync(frame.SequenceId)).FrameCount);
+ }
+
+ private static byte[] Photo(int seed = 0)
{
const int width = 96, height = 64;
var pixels = new byte[width * height * 3];
- for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)(i * 7 % 251);
+ for (var i = 0; i < pixels.Length; i++) pixels[i] = (byte)((i * 7 + seed * 31) % 251);
return JpegEncoder.Encode(pixels, width, height, 3, new JpegEncodeOptions { Quality = 90 });
}
}
diff --git a/tests/Titano.Server.Tests/Storage/UploadTests.cs b/tests/Titano.Server.Tests/Storage/UploadTests.cs
index ae889782..aea2324e 100644
--- a/tests/Titano.Server.Tests/Storage/UploadTests.cs
+++ b/tests/Titano.Server.Tests/Storage/UploadTests.cs
@@ -1,6 +1,7 @@
using System.Net;
using System.Net.Http.Json;
using Microsoft.Extensions.DependencyInjection;
+using Titano.Core.Model;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Server.Storage;
@@ -96,6 +97,49 @@ public sealed class UploadTests(TestContext context)
Assert.Equal(frame.FrameId, again.FrameId);
}
+ [Test]
+ public async Task ACopyFlaggedOrGoneIsNoShortcut()
+ {
+ await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
+ var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
+ // A client may delete its file on "already exists": a damaged copy must ask for the bytes again.
+ var flagged = Uploads.Photo(5000, 11);
+ var first = await Uploads.UploadAsync(uploader, flagged);
+ await server.Database.Frames.UpdateFlagsAsync([first.FrameId], FrameFlags.Corrupt, FrameFlags.None);
+ Assert.False((await Uploads.StartAsync(uploader, flagged)).AlreadyExists);
+
+ var gone = Uploads.Photo(5000, 12);
+ var second = await Uploads.UploadAsync(uploader, gone);
+ File.Delete(Path.Combine(server.Options.ArchivePath, Assert.NotNull(await server.Database.Frames.GetAsync(second.FrameId)).RelativePath));
+ Assert.False((await Uploads.StartAsync(uploader, gone)).AlreadyExists);
+ }
+
+ [Test]
+ public async Task ACommitLeftByACrashIsSettledOnlyByAnIntactFrame()
+ {
+ await using var server = await TestServer.StartAsync(context, o => o.ChunkSizeBytes = Chunk);
+ var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
+ var data = Uploads.Photo(Chunk + 100, 13);
+ var missing = await Uploads.UploadAsync(uploader, data);
+ await server.Database.Frames.UpdateFlagsAsync([missing.FrameId], FrameFlags.Missing, FrameFlags.None);
+ var start = await Uploads.StartAsync(uploader, data);
+ var id = Assert.NotNull(start.UploadId);
+ for (var i = 0; i < 2; i++)
+ {
+ using var put = await Uploads.PutChunkAsync(uploader, id, i, Uploads.Chunk(data, Chunk, i));
+ await put.ExpectAsync(HttpStatusCode.OK);
+ }
+ // The server stopped while committing it.
+ Assert.True(await server.Database.Uploads.TryTransitionAsync(id, Titano.Data.Repositories.UploadState.Receiving,
+ Titano.Data.Repositories.UploadState.Committing, cancellationToken: context.Cancellation));
+
+ await server.Services.GetRequiredService().RecoverAsync(context.Cancellation);
+
+ // The flagged frame does not stand for it: the client completes again with its part file intact.
+ using (var state = await uploader.GetAsync($"/api/v1/uploads/{id}")) Assert.Equal("receiving", (await state.ReadAsync(ProtocolJson.Default.UploadState)).State);
+ Assert.True(server.Services.GetRequiredService().Exists(id));
+ }
+
[Test]
public async Task AnUploadKeepsItsChunkSizeWhenTheSettingShrinks()
{