From 94a56278e239faa9455c166a7e617ea5793f8096 Mon Sep 17 00:00:00 2001 From: Alberto Balbo Date: Mon, 16 Nov 2026 19:56:30 +0100 Subject: [PATCH] feat(server): photo lookup endpoint and uploads into a target (owner review 6, P2) Clients must recognise the same time-lapse filed under another id on the server, and an integrated pair must get a photo into the server sequence even when another sequence there already holds it. The server now answers both, authorised and scoped by itself. - POST /api/v1/photos/lookup (Read right): per probe, the sequences of the served archive that hold its hashes in a frame, tombstoned them (deleted or replaced by a conversion), match its unhashed photos by size and capture millisecond, or carry its session key; only ids and counts, batched raw SQL, 413 too_many beyond 200 probes or 20,000 items, a 4 MB body limit so a full request fits. - Uploads with IntoTarget: sequenceId required, never with Restore, the target must be the served archive's and the caller its owner, an admin or an editor. Only a copy in the target stands in; a tombstone keeps the photo deleted. Otherwise the server copies an intact file of another sequence of the served archive (never another archive of the index) into the target: quota and room checked, verified copy, hash compared (a damaged source falls back to a normal upload), committed refusing removed photos, used bytes charged only for a new frame, frames event published, answered copied. - server/info announces photo-lookup and upload-into (the desktop's hosted share too). Refs #73 Co-Authored-By: Claude Opus 5.5 --- .../Api/PhotoLookupEndpoints.cs | 46 +++ src/Chronos.Server/Api/SystemEndpoints.cs | 3 +- src/Chronos.Server/Hosting/ServerHost.cs | 2 + src/Chronos.Server/Storage/PhotoLookup.cs | 205 ++++++++++++ src/Chronos.Server/Storage/UploadRules.cs | 24 +- src/Chronos.Server/Storage/UploadService.cs | 116 ++++++- .../Api/PhotoLookupTests.cs | 239 ++++++++++++++ .../Storage/UploadIntoTests.cs | 298 ++++++++++++++++++ 8 files changed, 926 insertions(+), 7 deletions(-) create mode 100644 src/Chronos.Server/Api/PhotoLookupEndpoints.cs create mode 100644 src/Chronos.Server/Storage/PhotoLookup.cs create mode 100644 tests/Chronos.Server.Tests/Api/PhotoLookupTests.cs create mode 100644 tests/Chronos.Server.Tests/Storage/UploadIntoTests.cs diff --git a/src/Chronos.Server/Api/PhotoLookupEndpoints.cs b/src/Chronos.Server/Api/PhotoLookupEndpoints.cs new file mode 100644 index 00000000..9903bd7b --- /dev/null +++ b/src/Chronos.Server/Api/PhotoLookupEndpoints.cs @@ -0,0 +1,46 @@ +using System.Globalization; +using Microsoft.AspNetCore.Http.Metadata; +using Chronos.Core.Model; +using Chronos.Protocol; +using Chronos.Server.Security; +using Chronos.Server.Storage; + +namespace Chronos.Server.Api; + +/// +/// POST /api/v1/photos/lookup (owner review 6, #73; capability ): which +/// sequences of the served archive hold photos a client sampled from its own sequences (). Needs +/// the Read right; at most probes and +/// items, 413 too_many beyond. +/// +public static class PhotoLookupEndpoints +{ + /// + /// The body a full request may take: 20,000 items of about 70 bytes each are beyond the small limit of the other JSON + /// endpoints (). + /// + private const long MaxBodyBytes = 4L * 1024 * 1024; + + public static void Map(RouteGroupBuilder api) + { + ArgumentNullException.ThrowIfNull(api); + api.MapPost("/photos/lookup", LookupAsync).Requires(Permissions.Read).WithMetadata(new LookupBodyLimit()); + } + + private static async Task LookupAsync(HttpContext context, PhotoLookupRequest request, PhotoLookup lookup) + { + if (request.Probes is null) return ApiErrors.BadRequest(context, "bad_request", "probes"); + if (PhotoLookup.TooMany(request)) + { + return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_many", + PhotoLookupRequest.MaxItems.ToString(CultureInfo.InvariantCulture)); + } + return Results.Json(await lookup.LookupAsync(request, context.RequestAborted), ProtocolJson.Default.PhotoLookupResponse); + } + + /// The lookup's body limit, applied by routing as it selects the endpoint (as the chunk route's is). + private sealed class LookupBodyLimit : IRequestSizeLimitMetadata + { + public long? MaxRequestBodySize => MaxBodyBytes; + } +} diff --git a/src/Chronos.Server/Api/SystemEndpoints.cs b/src/Chronos.Server/Api/SystemEndpoints.cs index 7fcfd00e..0a9d44a3 100644 --- a/src/Chronos.Server/Api/SystemEndpoints.cs +++ b/src/Chronos.Server/Api/SystemEndpoints.cs @@ -125,7 +125,8 @@ public sealed class ServerState public string Database { get; init; } = string.Empty; /// What this server offers clients (; names those clients act on). - public List Capabilities { get; } = ["uploads", "sync", "jobs", "render-nodes", "events", ServerCapabilities.Projects, ServerCapabilities.Videos]; + public List Capabilities { get; } = ["uploads", "sync", "jobs", "render-nodes", "events", ServerCapabilities.Projects, ServerCapabilities.Videos, + ServerCapabilities.PhotoLookup, ServerCapabilities.UploadInto]; public DateTimeOffset StartedAt { get; } = DateTimeOffset.UtcNow; } diff --git a/src/Chronos.Server/Hosting/ServerHost.cs b/src/Chronos.Server/Hosting/ServerHost.cs index e9bbcbfc..796b31d5 100644 --- a/src/Chronos.Server/Hosting/ServerHost.cs +++ b/src/Chronos.Server/Hosting/ServerHost.cs @@ -245,6 +245,7 @@ public static class ServerHost services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -311,6 +312,7 @@ public static class ServerHost AuthEndpoints.Map(api); UserEndpoints.Map(api); SequenceEndpoints.Map(api); + PhotoLookupEndpoints.Map(api); ProjectEndpoints.Map(api); LibraryEndpoints.Map(api); FileEndpoints.Map(api); diff --git a/src/Chronos.Server/Storage/PhotoLookup.cs b/src/Chronos.Server/Storage/PhotoLookup.cs new file mode 100644 index 00000000..ef83588d --- /dev/null +++ b/src/Chronos.Server/Storage/PhotoLookup.cs @@ -0,0 +1,205 @@ +using System.Globalization; +using System.Text; +using Chronos.Data; +using Chronos.Protocol; + +namespace Chronos.Server.Storage; + +/// +/// POST /api/v1/photos/lookup (owner review 6, #73): which sequences of the served archive hold the photos a client +/// sampled from its own sequences, so it recognises the same time-lapse filed under another id. Each probe's hashes are +/// looked up in the frames and the tombstones, its unhashed photos by size and capture time to the millisecond, and its +/// back-reference key in the session keys. The answer names only sequence ids and counts, never files or titles; another +/// archive of the index is never looked at. +/// +public sealed class PhotoLookup(ChronosDatabase db, ServerArchive archive) +{ + /// Parameters of one statement: the archive id and the values of one batch. + private const int BatchParameters = 500; + + private const int BatchValues = BatchParameters - 1; + + /// The unhashed range query binds the archive and the two ends of the range besides the sizes. + private const int UnhashedBatchValues = BatchParameters - 3; + + /// + /// The request goes beyond probes or + /// items (413 too_many). + /// + public static bool TooMany(PhotoLookupRequest request) + { + ArgumentNullException.ThrowIfNull(request); + var probes = request.Probes ?? []; + if (probes.Count > PhotoLookupRequest.MaxProbes) return true; + long items = 0; + foreach (var probe in probes) + { + if (probe is null) continue; + items += (probe.Sha256?.Count ?? 0) + (probe.Unhashed?.Count ?? 0) + (probe.SessionKey is null ? 0 : 1); + } + return items > PhotoLookupRequest.MaxItems; + } + + /// One result per probe, in the request's order; a probe that hit nothing has no hits. + public async Task LookupAsync(PhotoLookupRequest request, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(request); + var probes = (request.Probes ?? []).Select(Normalize).ToList(); + var hashes = probes.SelectMany(p => p.Hashes).Distinct(StringComparer.Ordinal).ToList(); + var keys = probes.SelectMany(p => p.Unhashed).Distinct().ToList(); + var sessions = probes.Where(p => p.SessionKey is not null).Select(p => p.SessionKey!).Distinct(StringComparer.Ordinal).ToList(); + + var held = new Dictionary>(StringComparer.Ordinal); + var removed = new Dictionary>(StringComparer.Ordinal); + var byKey = new Dictionary>(); + var bySession = new Dictionary>(StringComparer.Ordinal); + + var archiveId = archive.Id; + await using (var session = await db.Database.OpenAsync(ct).ConfigureAwait(false)) + { + foreach (var batch in hashes.Chunk(BatchValues)) + { + var (list, parameters) = InList(archiveId, batch); + await foreach (var row in session.QueryAsync( + "SELECT f.sequence_id, f.sha256 FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + $"WHERE s.archive_id = $1 AND f.sha256 IN ({list})", parameters, ct).ConfigureAwait(false)) + Add(held, row.GetString(1), row.GetGuid(0)); + } + var wanted = hashes.ToHashSet(StringComparer.Ordinal); + foreach (var batch in hashes.Chunk(BatchValues)) + { + var (list, parameters) = InList(archiveId, batch); + await foreach (var row in session.QueryAsync( + "SELECT r.sequence_id, r.sha256, r.replacement_sha256 FROM frame_removals r JOIN sequences s ON s.id = r.sequence_id " + + $"WHERE s.archive_id = $1 AND (r.sha256 IN ({list}) OR r.replacement_sha256 IN ({list}))", parameters, ct).ConfigureAwait(false)) + { + var sequence = row.GetGuid(0); + if (wanted.Contains(row.GetString(1))) Add(removed, row.GetString(1), sequence); + if (row.GetStringOrNull(2) is { } replacement && wanted.Contains(replacement)) Add(removed, replacement, sequence); + } + } + // Sorted by capture time, so each batch asks for a narrow range of the frames. + foreach (var batch in keys.OrderBy(k => k.CaptureTicks).Chunk(UnhashedBatchValues)) + { + var wantedKeys = batch.ToHashSet(); + var parameters = new List + { + archiveId, + new(new DateTime(batch[0].CaptureTicks, DateTimeKind.Unspecified), DbValueType.LocalDateTime), + new(new DateTime(batch[^1].CaptureTicks + TimeSpan.TicksPerMillisecond - 1, DateTimeKind.Unspecified), DbValueType.LocalDateTime), + }; + var sizes = new StringBuilder(); + foreach (var size in batch.Select(k => k.Size).Distinct()) + { + parameters.Add(size); + if (sizes.Length > 0) sizes.Append(", "); + sizes.Append('$').Append(parameters.Count.ToString(CultureInfo.InvariantCulture)); + } + await foreach (var row in session.QueryAsync( + "SELECT f.sequence_id, f.size, f.captured_at FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + $"WHERE s.archive_id = $1 AND f.captured_at >= $2 AND f.captured_at <= $3 AND f.size IN ({sizes})", + [.. parameters], ct).ConfigureAwait(false)) + { + if (row.GetDateTimeOrNull(2) is not { } captured) continue; + var key = new PhotoKey(row.GetInt64(1), Millisecond(captured)); + if (wantedKeys.Contains(key)) Add(byKey, key, row.GetGuid(0)); + } + } + foreach (var batch in sessions.Chunk(BatchValues)) + { + var (list, parameters) = InList(archiveId, batch); + await foreach (var row in session.QueryAsync( + $"SELECT session_key, sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key IN ({list})", + parameters, ct).ConfigureAwait(false)) + Add(bySession, row.GetString(0), row.GetGuid(1)); + } + } + + var results = new List(probes.Count); + foreach (var probe in probes) + { + var counts = new Dictionary(); + foreach (var hash in probe.Hashes) + { + var holders = held.GetValueOrDefault(hash); + foreach (var sequence in holders ?? []) Get(counts, sequence).Hashes++; + // A tombstone of content the sequence still holds in a frame is stale: the photo counts once, as held. + foreach (var sequence in removed.GetValueOrDefault(hash) ?? []) + { + if (holders?.Contains(sequence) != true) Get(counts, sequence).Removed++; + } + } + foreach (var key in probe.Unhashed) + { + foreach (var sequence in byKey.GetValueOrDefault(key) ?? []) Get(counts, sequence).Unhashed++; + } + if (probe.SessionKey is { } sessionKey) + { + foreach (var sequence in bySession.GetValueOrDefault(sessionKey) ?? []) Get(counts, sequence).SessionKey = true; + } + results.Add(new ProbeResult(probe.Key, [.. counts + .OrderByDescending(c => (c.Value.SessionKey ? 1 : 0, c.Value.Hashes + c.Value.Removed + c.Value.Unhashed)) + .ThenBy(c => c.Key) + .Select(c => new PhotoLookupHit(c.Key, c.Value.Hashes, c.Value.Removed, c.Value.Unhashed, c.Value.SessionKey))])); + } + return new PhotoLookupResponse(results); + } + + /// A probe as a set: lower-case hashes, keys to the millisecond, a trimmed session key; each counted once. + private static Probe Normalize(PhotoProbe? probe) => + probe is null + ? new Probe(string.Empty, [], [], null) + : new Probe( + probe.Key ?? string.Empty, + [.. (probe.Sha256 ?? []).Where(h => h is { Length: 64 }).Select(h => h.ToLowerInvariant()).Distinct(StringComparer.Ordinal)], + [.. (probe.Unhashed ?? []).Where(k => k is not null).Select(k => new PhotoKey(k.Size, Millisecond(k.CapturedAt))).Distinct()], + string.IsNullOrWhiteSpace(probe.SessionKey) ? null : probe.SessionKey.Trim()); + + /// + /// A capture time to the millisecond, as a wall-clock time whatever kind it came with (as + /// SequenceRepository.Wall stores it): the index keeps microseconds, a client may send ticks. + /// + private static long Millisecond(DateTime value) => value.Ticks - value.Ticks % TimeSpan.TicksPerMillisecond; + + /// $2, $3, … for , after the archive id as $1. + private static (string List, DbParam[] Parameters) InList(Guid archiveId, string[] values) + { + var parameters = new DbParam[values.Length + 1]; + parameters[0] = archiveId; + var list = new StringBuilder(); + for (var i = 0; i < values.Length; i++) + { + parameters[i + 1] = values[i]; + if (i > 0) list.Append(", "); + list.Append('$').Append((i + 2).ToString(CultureInfo.InvariantCulture)); + } + return (list.ToString(), parameters); + } + + private static void Add(Dictionary> map, TKey key, Guid sequence) where TKey : notnull + { + if (!map.TryGetValue(key, out var set)) map[key] = set = []; + set.Add(sequence); + } + + private static Counts Get(Dictionary counts, Guid sequence) + { + if (!counts.TryGetValue(sequence, out var found)) counts[sequence] = found = new Counts(); + return found; + } + + private readonly record struct PhotoKey(long Size, long CaptureTicks); + + private sealed record Probe(string Key, IReadOnlyList Hashes, IReadOnlyList Unhashed, string? SessionKey); + + private sealed class Counts + { + public int Hashes { get; set; } + + public int Removed { get; set; } + + public int Unhashed { get; set; } + + public bool SessionKey { get; set; } + } +} diff --git a/src/Chronos.Server/Storage/UploadRules.cs b/src/Chronos.Server/Storage/UploadRules.cs index 687fbf95..188211fa 100644 --- a/src/Chronos.Server/Storage/UploadRules.cs +++ b/src/Chronos.Server/Storage/UploadRules.cs @@ -47,13 +47,31 @@ public static class UploadRules /// /// The frame that stands for an upload of content the archive already has (): an intact copy, /// the one in the upload's sequence first. Null when there is none, or when the target holds - /// the content only in damaged copies: the upload's bytes repair them, whatever other sequences hold. + /// the content only in damaged copies: the upload's bytes repair them, whatever other sequences hold. With + /// (StartUploadRequest.IntoTarget, owner review 6, #73) only an intact copy in the + /// target stands in: the photo is to be filed there, a copy elsewhere is no answer. /// - public static Frame? StandIn(string archiveRoot, IReadOnlyList copies, Guid? target) + public static Frame? StandIn(string archiveRoot, IReadOnlyList copies, Guid? target, bool intoTarget = false) { var intact = copies.Where(f => IsIntact(archiveRoot, f)).ToList(); if (copies.Any(f => f.SequenceId == target) && !intact.Any(f => f.SequenceId == target)) return null; - return intact.FirstOrDefault(f => f.SequenceId == target) ?? intact.FirstOrDefault(); + return intact.FirstOrDefault(f => f.SequenceId == target) ?? (intoTarget ? null : intact.FirstOrDefault()); + } + + /// + /// The file the server copies into for an upload into it (owner review 6, #73): an intact copy + /// held by another sequence of the served archive , resolved under that archive's + /// . A frame of a sequence of another archive of the index ( + /// names each copy's sequence's archive; a desktop sharing its library keeps other archives' frames in the same index) + /// is never a source. Null when there is none. + /// + public static Frame? CopySource(string archiveRoot, IReadOnlyList copies, Guid target, Guid archiveId, + IReadOnlyDictionary archiveOf) + { + ArgumentNullException.ThrowIfNull(copies); + ArgumentNullException.ThrowIfNull(archiveOf); + return copies.FirstOrDefault(f => f.SequenceId != target && archiveOf.TryGetValue(f.SequenceId, out var archive) && archive == archiveId + && IsIntact(archiveRoot, f)); } /// diff --git a/src/Chronos.Server/Storage/UploadService.cs b/src/Chronos.Server/Storage/UploadService.cs index 1455256d..6d5c7434 100644 --- a/src/Chronos.Server/Storage/UploadService.cs +++ b/src/Chronos.Server/Storage/UploadService.cs @@ -2,6 +2,8 @@ using System.Collections.Concurrent; using System.Globalization; using Microsoft.AspNetCore.Http.Features; using Chronos.Core.Archiving; +using Chronos.Core.Import; +using Chronos.Core.Model; using Chronos.Data; using Chronos.Data.Archiving; using Chronos.Data.Repositories; @@ -56,7 +58,18 @@ public sealed class UploadService( if (request.Size <= 0) return Fail(400, "bad_request", "size"); if (request.Size > options.MaxUploadBytes) return Fail(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture)); if (request.SessionKey is { Length: > 200 }) return Fail(400, "bad_request", "sessionKey"); - if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null) + if (request.IntoTarget) + { + // Authorised here, never trusted to the client (owner review 6, #73): the served archive's sequence, which the + // caller owns or may edit, and never a tombstone dropped through a copy the server makes itself. + if (request.SequenceId is not { } into) return Fail(400, "bad_request", "sequenceId"); + if (request.Restore) return Fail(400, "bad_request", "restore"); + if (await ServedItems.SequenceAsync(db, archive, into, cancellationToken) is not { } intoSequence) + return Fail(404, "not_found", "sequence"); + if (intoSequence.OwnerUserId != caller.Id && !caller.IsAdmin && !caller.Has(Permissions.Edit)) + return Fail(403, "forbidden"); + } + else if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null) return Fail(404, "not_found", "sequence"); var sha = request.Sha256.ToLowerInvariant(); @@ -70,9 +83,16 @@ public sealed class UploadService( var removed = await UploadRules.RemovedFromTargetAsync(db, sha, target, copies, cancellationToken); if (removed && !request.Restore) return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, null); // Only a copy that is still whole stands for the upload: a client may delete its file on this answer. A photo - // brought back is filed into its sequence, whatever other sequences hold it. - if (UploadRules.StandIn(archive.Root, removed ? [] : copies, target) is { } known) + // brought back is filed into its sequence, whatever other sequences hold it. An upload into the target hears only of + // a copy there. + if (UploadRules.StandIn(archive.Root, removed ? [] : copies, target, request.IntoTarget) is { } known) return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id); + // Into a target that holds no copy, while another sequence of the served archive holds an intact one: the server + // files its own file there, no bytes on the wire. A copy that does not read back as the request's content leaves + // the client to send the bytes. + if (request.IntoTarget && target is { } targetId && !copies.Any(f => f.SequenceId == targetId) + && await CopyIntoAsync(caller, sha, targetId, copies, cancellationToken) is { } copied) + return copied; var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken); if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id)) @@ -325,6 +345,96 @@ public sealed class UploadService( }); } + /// + /// Files into the server's own file of held by another sequence of the + /// served archive (, owner review 6, #73): checked against the caller's quota + /// and the archive's room, copied to staging and read back (), its SHA-256 compared with the + /// request's, and committed as an upload of it would be, refusing a photo the target deleted. Used bytes grow only when + /// a new frame was filed. Null when there is no such file, or when the copy is not the request's content (a damaged + /// source): the client then sends the bytes. + /// + private async Task?> CopyIntoAsync(Caller caller, string sha, Guid target, IReadOnlyList copies, + CancellationToken cancellationToken) + { + var archiveOf = new Dictionary(); + foreach (var sequenceId in copies.Select(f => f.SequenceId).Distinct()) + { + if (await db.Sequences.GetAsync(sequenceId, cancellationToken) is { } sequence) archiveOf[sequenceId] = sequence.ArchiveId; + } + if (UploadRules.CopySource(archive.Root, copies, target, archive.Id, archiveOf) is not { } source + || ArchivePathGuard.ExistingFile(archive.Root, source.RelativePath) is not { } file) + return null; + + if (caller.User.QuotaBytes > 0) + { + var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken); + if (caller.User.UsedBytes + pending + source.Size > caller.User.QuotaBytes) + return Fail(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture)); + } + if (volumes.Measure(archive.Root) is { } volume && volume.FreeBytes < source.Size + DiskReserve) + return Fail(507, "insufficient_storage"); + + // Runs to the end once started, even if the client disconnects, as a completion does. + var stopping = lifetime.ApplicationStopping; + var staged = Path.Combine(options.StagingPath, Guid.CreateVersion7().ToString("N") + Path.GetExtension(source.FileName).ToLowerInvariant()); + try + { + var copied = await VerifiedCopy.CopyAsync(file.FullName, staged, verify: true, cancellationToken: stopping); + if (copied != sha) + { + logger.LogWarning("Frame {Frame} ({File}) holds SHA-256 {Actual} instead of {Expected}: not copied into sequence {Target}, the client sends the photo", + source.Id, source.RelativePath, copied, sha, target); + UploadStore.TryDelete(staged); + return null; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + logger.LogWarning(ex, "Frame {Frame} ({File}) could not be copied into sequence {Target}: the client sends the photo", source.Id, source.RelativePath, target); + UploadStore.TryDelete(staged); + return null; + } + + IngestResult result; + try + { + result = await ingest.CommitAsync(new IngestRequest(archive.Id, target, NewSequenceTitle: null, SessionKey: null, staged, + source.OriginalFileName ?? source.FileName, source.Size, sha, source.CapturedAt, caller.Id, RefuseRemoved: true), stopping); + } + catch (PhotoRemovedException) + { + // Deleted from the target since the start checked: it stays deleted. + UploadStore.TryDelete(staged); + return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, null); + } + catch (Exception ex) when (ex is LeaseUnavailableException or ArchiveOfflineException or SequenceFolderMovedException or FolderMovedException) + { + logger.LogWarning("Frame {Frame} not copied into sequence {Target} for now: {Reason}", source.Id, target, ex.Message); + UploadStore.TryDelete(staged); + return Fail(503, ex switch + { + ArchiveOfflineException => "archive_offline", + SequenceFolderMovedException or FolderMovedException => "folder_moved", + _ => "sequence_busy", + }); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogError(ex, "Copying frame {Frame} into sequence {Target} failed", source.Id, target); + UploadStore.TryDelete(staged); + return Fail(503, "commit_failed"); + } + + if (File.Exists(staged)) UploadStore.TryDelete(staged); + if (!result.Duplicate && !result.Repaired) await db.Users.AddUsedBytesAsync(caller.Id, source.Size, CancellationToken.None); + if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } filed) + { + await sequenceJobs.FramesAddedAsync(filed, CancellationToken.None); + events.Publish(new SequenceChangedEvent(filed.Id, "frames", filed.FramesVersion, filed.SettingsVersion)); + } + return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, result.FrameId, Copied: true); + } + private async Task FailAsync(Upload upload) { await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None); diff --git a/tests/Chronos.Server.Tests/Api/PhotoLookupTests.cs b/tests/Chronos.Server.Tests/Api/PhotoLookupTests.cs new file mode 100644 index 00000000..f7d18e0e --- /dev/null +++ b/tests/Chronos.Server.Tests/Api/PhotoLookupTests.cs @@ -0,0 +1,239 @@ +using System.Net; +using System.Net.Http.Json; +using Chronos.Core.Model; +using Chronos.Data; +using Chronos.Protocol; +using Chronos.Server.Configuration; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Api; + +/// POST /api/v1/photos/lookup (owner review 6, #73): counts per probe, scoped to the served archive. +[Category("server")] +public sealed class PhotoLookupTests(TestContext context) +{ + private static readonly DateTime Shot = new DateTime(2026, 9, 28, 18, 2, 11, 123, DateTimeKind.Unspecified).AddTicks(4560); + + [Test] + public async Task EachProbeCountsTheSequencesThatHoldItsPhotos() + { + await using var server = await TestServer.StartAsync(context); + await CountsAsync(server, context); + } + + [Test] + [RequiresEnvironment("CHRONOS_TEST_PG")] + public async Task EachProbeCountsTheSequencesThatHoldItsPhotosOnPostgres() + { + await using var schema = await PgSchema.CreateAsync(context); + await using var server = await schema.StartAsync(context); + Assert.Equal(Dialect.Postgres, server.Database.Dialect); + await CountsAsync(server, context); + } + + [Test] + public async Task OnlyTheServedArchiveIsLookedAt() + { + await using var server = await TestServer.StartAsync(context); + await ScopedAsync(server, context); + } + + [Test] + [RequiresEnvironment("CHRONOS_TEST_PG")] + public async Task OnlyTheServedArchiveIsLookedAtOnPostgres() + { + await using var schema = await PgSchema.CreateAsync(context); + await using var server = await schema.StartAsync(context); + await ScopedAsync(server, context); + } + + [Test] + public async Task TheLookupNeedsTheReadRight() + { + await using var server = await TestServer.StartAsync(context); + var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader"); + using (var refused = await LookupAsync(uploader, new PhotoLookupRequest([new PhotoProbe("a", [Hash('a')])]))) + await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); + using (var anonymous = await LookupAsync(server.Client, new PhotoLookupRequest([new PhotoProbe("a", [Hash('a')])]))) + Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode); + var (viewer, _, _) = await server.UserAsync("guest", "viewer"); + using var allowed = await LookupAsync(viewer, new PhotoLookupRequest([new PhotoProbe("a", [Hash('a')])])); + Assert.Empty(Assert.NotNull(await allowed.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Single().Hits); + } + + [Test] + public async Task MoreThan200ProbesOr20000ItemsAreRefused() + { + await using var server = await TestServer.StartAsync(context); + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + + var most = Enumerable.Range(0, PhotoLookupRequest.MaxProbes).Select(i => new PhotoProbe("p" + i, [Hash('b')])).ToList(); + using (var ok = await LookupAsync(editor, new PhotoLookupRequest(most))) + Assert.Equal(PhotoLookupRequest.MaxProbes, (await ok.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Count); + using (var probes = await LookupAsync(editor, new PhotoLookupRequest([.. most, new PhotoProbe("one more", [Hash('b')])]))) + await probes.ExpectErrorAsync(HttpStatusCode.RequestEntityTooLarge, "too_many"); + + // 20,000 items fit in one request (a body larger than the other JSON calls take); one more does not. + var hashes = Enumerable.Range(0, PhotoLookupRequest.MaxItems - 100).Select(i => i.ToString("x64", System.Globalization.CultureInfo.InvariantCulture)).ToList(); + var unhashed = Enumerable.Range(0, 99).Select(i => new PhotoKeyDto(1000 + i, Shot.AddSeconds(i))).ToList(); + var full = new PhotoProbe("full", hashes, unhashed, "chronos-sync:" + Guid.NewGuid().ToString("N")); + Assert.Equal(PhotoLookupRequest.MaxItems, new PhotoLookupRequest([full]).CountItems()); + using (var ok = await LookupAsync(editor, new PhotoLookupRequest([full]))) + Assert.Empty((await ok.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Single().Hits); + var over = full with { Unhashed = [.. unhashed, new PhotoKeyDto(5, Shot)] }; + using (var items = await LookupAsync(editor, new PhotoLookupRequest([over]))) + await items.ExpectErrorAsync(HttpStatusCode.RequestEntityTooLarge, "too_many"); + } + + [Test] + public async Task TheServerAnnouncesTheLookupAndUploadsIntoATarget() + { + await using var server = await TestServer.StartAsync(context); + using var response = await server.Client.GetAsync("/api/v1/server/info"); + var info = await response.ReadAsync(ProtocolJson.Default.ServerInfo); + Assert.Contains(ServerCapabilities.PhotoLookup, info.Capabilities); + Assert.Contains(ServerCapabilities.UploadInto, info.Capabilities); + } + + /// + /// Sequence A holds p1, p2 and p3 (p3 captured at ); B holds p4, tombstoned p5, replaced p6 by p7 and + /// is filed under a session key. One probe counts both, another hits nothing. + /// + internal static async Task CountsAsync(TestServer server, TestContext context) + { + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var photos = Enumerable.Range(1, 7).Select(i => Uploads.Photo(3000 + i, 700 + i)).ToArray(); + var a = (await Uploads.UploadAsync(editor, photos[0], "IMG_0001.JPG", sessionKey: "a")).SequenceId; + await Uploads.UploadAsync(editor, photos[1], "IMG_0002.JPG", sessionKey: "a"); + await Uploads.UploadAsync(editor, photos[2], "IMG_0003.JPG", sessionKey: "a", capturedAt: Shot); + var b = (await Uploads.UploadAsync(editor, photos[3], "IMG_0004.JPG", sessionKey: "b")).SequenceId; + var now = DateTimeOffset.UtcNow; + await server.Database.FrameRemovals.SyncSequenceAsync(b, + [ + new FrameRemoval(b, Uploads.Sha(photos[4]), null, RemovalReason.Deleted, now), + new FrameRemoval(b, Uploads.Sha(photos[5]), Uploads.Sha(photos[6]), RemovalReason.Replaced, now), + // A stale tombstone: B still holds p4, which counts once, as held. + new FrameRemoval(b, Uploads.Sha(photos[3]), null, RemovalReason.Deleted, now), + ], context.Cancellation); + var archive = (await server.Database.Sequences.GetAsync(b, context.Cancellation))!.ArchiveId; + var key = "chronos-sync:" + Guid.NewGuid().ToString("N"); + await server.Database.Sequences.MapSessionKeyAsync(archive, key, b, context.Cancellation); + + var probe = new PhotoProbe("local-1", + [Uploads.Sha(photos[0]).ToUpperInvariant(), Uploads.Sha(photos[1]), Uploads.Sha(photos[1]), Uploads.Sha(photos[3]), Uploads.Sha(photos[4]), + Uploads.Sha(photos[6]), Hash('c')], + [ + // The same millisecond as p3 (the index keeps microseconds); the next millisecond, another size: no match. + new PhotoKeyDto(photos[2].Length, new DateTime(2026, 9, 28, 18, 2, 11, 123).AddTicks(9999)), + new PhotoKeyDto(photos[2].Length, new DateTime(2026, 9, 28, 18, 2, 11, 124)), + new PhotoKeyDto(photos[2].Length + 1, Shot), + ], + key); + var nothing = new PhotoProbe("local-2", [Hash('d')], [new PhotoKeyDto(photos[2].Length, Shot.AddDays(1))], "chronos-sync:" + Guid.NewGuid().ToString("N")); + using var response = await LookupAsync(editor, new PhotoLookupRequest([probe, nothing])); + var results = (await response.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results; + + Assert.SequenceEqual(["local-1", "local-2"], results.Select(r => r.Key)); + var hits = results[0].Hits.ToDictionary(h => h.SequenceId); + Assert.Equal(2, hits.Count); + Assert.Equal(new PhotoLookupHit(a, Hashes: 2, Removed: 0, Unhashed: 1, SessionKey: false), hits[a]); + Assert.Equal(new PhotoLookupHit(b, Hashes: 1, Removed: 2, Unhashed: 0, SessionKey: true), hits[b]); + Assert.Empty(results[1].Hits); + } + + /// A sequence of another archive of the index holding the same photo and session key is never answered. + internal static async Task ScopedAsync(TestServer server, TestContext context) + { + var (editor, _, _) = await server.UserAsync("alberto", "editor"); + var photo = Uploads.Photo(4000, 801); + var mine = (await Uploads.UploadAsync(editor, photo, sessionKey: "mine", capturedAt: Shot)).SequenceId; + var key = "chronos-sync:" + Guid.NewGuid().ToString("N"); + var foreign = await ForeignFrameAsync(server, context, photo, Shot, key); + + using var response = await LookupAsync(editor, new PhotoLookupRequest([new PhotoProbe("x", [Uploads.Sha(photo)], [new PhotoKeyDto(photo.Length, Shot)], key)])); + var hit = (await response.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Single().Hits.Single(); + Assert.Equal(new PhotoLookupHit(mine, 1, 0, 1, false), hit); + Assert.NotEqual(foreign.SequenceId, hit.SequenceId); + } + + /// + /// A frame of in a sequence of a second archive of the index (as a desktop sharing its library + /// keeps other archives' sequences), its file under that archive's root; optionally filed under . + /// + internal static async Task ForeignFrameAsync(TestServer server, TestContext context, byte[] data, DateTime? capturedAt = null, string? sessionKey = null) + { + var db = server.Database; + var other = new Archive { Id = Guid.CreateVersion7(), Name = "Another archive", RootPath = context.CreateTempDirectory("other-archive"), Created = DateTimeOffset.UtcNow }; + await db.Archives.UpsertAsync(other, context.Cancellation); + var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = other.Id, Folder = "2026/other", UpdatedAt = DateTimeOffset.UtcNow }; + await db.Sequences.UpsertAsync(sequence, context.Cancellation); + const string relative = "2026/other/originals/0000_IMG_0001.CR3"; + var path = Path.Combine(other.RootPath, relative); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await File.WriteAllBytesAsync(path, data, context.Cancellation); + var frame = new Frame + { + Id = Guid.CreateVersion7(), + SequenceId = sequence.Id, + Index = 0, + RelativePath = relative, + FileName = Path.GetFileName(relative), + OriginalFileName = "IMG_0001.CR3", + Size = data.Length, + Sha256 = Uploads.Sha(data), + CapturedAt = capturedAt, + }; + await db.Frames.InsertAsync(frame, context.Cancellation); + await db.Sequences.RefreshStatsAsync(sequence.Id, context.Cancellation); + if (sessionKey is not null) await db.Sequences.MapSessionKeyAsync(other.Id, sessionKey, sequence.Id, context.Cancellation); + return frame; + } + + internal static Task LookupAsync(HttpClient client, PhotoLookupRequest request) => + client.PostAsJsonAsync("/api/v1/photos/lookup", request, ProtocolJson.Default.PhotoLookupRequest); + + private static string Hash(char c) => new(c, 64); +} + +/// A throw-away schema on the PostgreSQL test server (CHRONOS_TEST_PG), selected through Search Path. +internal sealed class PgSchema : IAsyncDisposable +{ + private readonly IDatabase _admin; + private readonly string _name; + + private PgSchema(IDatabase admin, string name, string connectionString) + { + _admin = admin; + _name = name; + ConnectionString = connectionString; + } + + public string ConnectionString { get; } + + public static async Task CreateAsync(TestContext context) + { + var baseConnection = Environment.GetEnvironmentVariable("CHRONOS_TEST_PG")!; + var admin = DatabaseFactory.Open(DatabaseOptions.Postgres(baseConnection)); + var name = "chronos_l_" + Guid.NewGuid().ToString("N")[..16]; + await using (var session = await admin.OpenAsync(context.Cancellation)) + await session.ExecuteScriptAsync($"CREATE SCHEMA {name}", context.Cancellation); + return new PgSchema(admin, name, baseConnection.TrimEnd(';', ' ') + $";Search Path={name}"); + } + + /// A test server on this schema. + public Task StartAsync(TestContext context, Action? configure = null, + Action? services = null) => + TestServer.StartAsync(context, o => + { + o.Database = DatabaseKind.Postgres; + o.DbConnectionString = ConnectionString; + configure?.Invoke(o); + }, services); + + public async ValueTask DisposeAsync() + { + await using (var session = await _admin.OpenAsync(CancellationToken.None)) + await session.ExecuteScriptAsync($"DROP SCHEMA IF EXISTS {_name} CASCADE", CancellationToken.None); + await _admin.DisposeAsync(); + } +} diff --git a/tests/Chronos.Server.Tests/Storage/UploadIntoTests.cs b/tests/Chronos.Server.Tests/Storage/UploadIntoTests.cs new file mode 100644 index 00000000..c4f2f7d6 --- /dev/null +++ b/tests/Chronos.Server.Tests/Storage/UploadIntoTests.cs @@ -0,0 +1,298 @@ +using System.Net; +using System.Net.Http.Json; +using Microsoft.Extensions.DependencyInjection; +using Chronos.Core.Model; +using Chronos.Data; +using Chronos.Data.Archiving; +using Chronos.Protocol; +using Chronos.Server.Events; +using Chronos.Server.Tests.Api; +using Chronos.Testing; + +namespace Chronos.Server.Tests.Storage; + +/// +/// (owner review 6, #73): a photo another sequence of the served archive holds is +/// copied into the target by the server itself, authorised and scoped on the server. +/// +[Category("server")] +public sealed class UploadIntoTests(TestContext context) +{ + private static readonly DateTime Shot = new(2026, 9, 28, 18, 2, 11, 250, DateTimeKind.Unspecified); + + [Test] + public async Task APhotoHeldElsewhereIsCopiedIntoTheTargetAndChargedOnce() + { + await using var server = await TestServer.StartAsync(context); + await CopiedAsync(server, context); + } + + [Test] + [RequiresEnvironment("CHRONOS_TEST_PG")] + public async Task APhotoHeldElsewhereIsCopiedIntoTheTargetAndChargedOnceOnPostgres() + { + await using var schema = await PgSchema.CreateAsync(context); + await using var server = await schema.StartAsync(context); + Assert.Equal(Dialect.Postgres, server.Database.Dialect); + await CopiedAsync(server, context); + } + + [Test] + public async Task ACommitThatFindsThePhotoFiledMeanwhileChargesNothing() + { + await using var server = await TestServer.StartAsync(context, services: s => + { + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + }); + var scene = await SceneAsync(server); + var racing = server.Services.GetRequiredService(); + Guid? raced = null; + // Another writer files the same photo into the target just before this commit: the ingest answers a duplicate. + racing.Before = async request => + { + var copy = request.StagedPath + ".race"; + File.Copy(request.StagedPath, copy); + raced = (await racing.Inner.CommitAsync(request with { StagedPath = copy }, context.Cancellation)).FrameId; + }; + var used = await UsedAsync(scene.Editor); + + using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target); + var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.True(answer.AlreadyExists && answer.Copied); + Assert.Equal(Assert.NotNull(raced), answer.FrameId); + Assert.Equal(used, await UsedAsync(scene.Editor)); + Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath)); + } + + [Test] + public async Task TheCopyIsRefusedBeyondTheQuota() + { + await using var server = await TestServer.StartAsync(context); + // The two uploads of the scene take 9,000 bytes; copying the 5,000-byte photo would take the account beyond 10,000. + var scene = await SceneAsync(server, quota: 10_000); + using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target); + await response.ExpectErrorAsync(HttpStatusCode.Forbidden, "quota_exceeded"); + Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo)); + } + + [Test] + public async Task OnlyTheOwnerAnAdminOrAnEditorMayUploadIntoATarget() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader"); + + // An upload-only account may not file into another account's sequence. + using (var refused = await StartIntoAsync(uploader, scene.Photo, scene.Target)) + await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden"); + Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo)); + + // Into its own sequence it may: the server copies the photo the editor's sequence holds. + var own = (await Uploads.UploadAsync(uploader, Uploads.Photo(3000, 903), "IMG_0200.CR3", sessionKey: "roof")).SequenceId; + using (var allowed = await StartIntoAsync(uploader, scene.Photo, own)) + Assert.True((await allowed.ReadAsync(ProtocolJson.Default.StartUploadResponse)).Copied); + Assert.Count(1, await FramesOfAsync(server, own, scene.Photo)); + + // An administrator may. + var admin = await server.AdminAsync(); + using (var byAdmin = await StartIntoAsync(admin, scene.Photo, scene.Target)) + Assert.True((await byAdmin.ReadAsync(ProtocolJson.Default.StartUploadResponse)).Copied); + } + + [Test] + public async Task ATargetOfAnotherArchiveIsUnknown() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + var foreign = await PhotoLookupTests.ForeignFrameAsync(server, context, Uploads.Photo(2000, 904)); + using (var other = await StartIntoAsync(scene.Editor, scene.Photo, foreign.SequenceId)) + Assert.Equal("sequence", (await other.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found")).Details); + using (var unknown = await StartIntoAsync(scene.Editor, scene.Photo, Guid.CreateVersion7())) + await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found"); + Assert.Empty(await FramesOfAsync(server, foreign.SequenceId, scene.Photo)); + } + + [Test] + public async Task AnUploadIntoATargetNamesItAndNeverRestores() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + using (var restore = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target, restore: true)) + Assert.Equal("restore", (await restore.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details); + using (var none = await StartIntoAsync(scene.Editor, scene.Photo, null)) + Assert.Equal("sequenceId", (await none.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details); + } + + [Test] + public async Task ATargetThatHoldsThePhotoAnswersItsFrame() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Held.SequenceId); + var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse); + Assert.True(answer.AlreadyExists); + Assert.False(answer.Copied); + Assert.Equal(scene.Held.FrameId, answer.FrameId); + } + + [Test] + public async Task APhotoTheTargetDeletedStaysDeleted() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + await server.Database.FrameRemovals.SyncSequenceAsync(scene.Target, + [new FrameRemoval(scene.Target, Uploads.Sha(scene.Photo), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], context.Cancellation); + var requests = server.Services.GetRequiredService().Requests.Count; + + using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target); + var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse); + Assert.True(answer.AlreadyExists); + Assert.Null(answer.FrameId); + Assert.False(answer.Copied); + Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo)); + Assert.Equal(requests, server.Services.GetRequiredService().Requests.Count); + } + + [Test] + public async Task APhotoOnlyAnotherArchiveHoldsIsUploaded() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + var elsewhere = Uploads.Photo(2500, 905); + await PhotoLookupTests.ForeignFrameAsync(server, context, elsewhere); + + using var response = await StartIntoAsync(scene.Editor, elsewhere, scene.Target); + Assert.Equal(HttpStatusCode.Created, response.StatusCode); + var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse); + Assert.NotNull(answer.UploadId); + Assert.False(answer.AlreadyExists || answer.Copied); + } + + [Test] + public async Task ADamagedSourceLeavesTheClientToSendThePhoto() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + // The file keeps its size, so the frame reads as intact; its bytes are no longer the photo. + var source = Assert.NotNull(await server.Database.Frames.GetAsync(scene.Held.FrameId)); + await File.WriteAllBytesAsync(Path.Combine(server.Options.ArchivePath, source.RelativePath), Uploads.Photo(scene.Photo.Length, 906), context.Cancellation); + + using (var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target)) + { + Assert.Equal(HttpStatusCode.Created, response.StatusCode); + var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse); + Assert.NotNull(answer.UploadId); + Assert.False(answer.Copied); + } + Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath)); + Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo)); + } + + [Test] + public async Task WithoutIntoTargetACopyElsewhereStillStandsIn() + { + await using var server = await TestServer.StartAsync(context); + var scene = await SceneAsync(server); + var used = await UsedAsync(scene.Editor); + var answer = await Uploads.StartAsync(scene.Editor, scene.Photo, sequenceId: scene.Target, expected: HttpStatusCode.OK); + Assert.True(answer.AlreadyExists); + Assert.False(answer.Copied); + Assert.Equal(scene.Held.FrameId, answer.FrameId); + Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo)); + Assert.Equal(used, await UsedAsync(scene.Editor)); + } + + /// The copy lands in the target with the photo's bytes, charged once, announced, and answered from the target after. + internal static async Task CopiedAsync(TestServer server, TestContext context) + { + var scene = await SceneAsync(server); + var used = await UsedAsync(scene.Editor); + using var events = server.Services.GetRequiredService().Subscribe(); + + StartUploadResponse answer; + using (var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target)) + { + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse); + } + Assert.True(answer.AlreadyExists && answer.Copied); + Assert.Null(answer.UploadId); + var frame = Assert.NotNull(await server.Database.Frames.GetAsync(Assert.NotNull(answer.FrameId), context.Cancellation)); + Assert.Equal(scene.Target, frame.SequenceId); + Assert.Equal(Uploads.Sha(scene.Photo), frame.Sha256); + await Http.BytesEqualAsync(scene.Photo, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, frame.RelativePath))); + Assert.NotNull(await server.Database.Frames.GetAsync(scene.Held.FrameId, context.Cancellation)); + var request = Assert.NotNull(server.Services.GetRequiredService().Requests.LastOrDefault()); + Assert.Equal(scene.Target, request.SequenceId); + Assert.True(request.RefuseRemoved, "the copy refuses a photo the target deleted"); + Assert.Equal(scene.Photo.Length, request.Size); + Assert.True(request.StagedPath.StartsWith(server.Options.StagingPath, StringComparison.Ordinal), request.StagedPath); + Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath)); + Assert.Equal(used + scene.Photo.Length, await UsedAsync(scene.Editor)); + + using (var wait = CancellationTokenSource.CreateLinkedTokenSource(context.Cancellation)) + { + wait.CancelAfter(TimeSpan.FromSeconds(10)); + while (await events.Reader.ReadAsync(wait.Token) is var e) + { + if (e is SequenceChangedEvent { Change: "frames" } changed && changed.SequenceId == scene.Target) break; + } + } + + // Asked again, the target answers with its own copy, charged nothing more. + using (var again = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target)) + { + var repeated = await again.ReadAsync(ProtocolJson.Default.StartUploadResponse); + Assert.True(repeated.AlreadyExists); + Assert.False(repeated.Copied); + Assert.Equal(frame.Id, repeated.FrameId); + } + Assert.Equal(used + scene.Photo.Length, await UsedAsync(scene.Editor)); + Assert.Count(1, await FramesOfAsync(server, scene.Target, scene.Photo)); + } + + /// An editor's sequence A holding a 5,000-byte photo, and the editor's sequence B (the target) holding another photo. + private static async Task SceneAsync(TestServer server, long quota = 0) + { + var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: quota); + var photo = Uploads.Photo(5000, 901); + var held = await Uploads.UploadAsync(editor, photo, "IMG_0001.CR3", sessionKey: "a", capturedAt: Shot); + var target = (await Uploads.UploadAsync(editor, Uploads.Photo(4000, 902), "IMG_0100.CR3", sessionKey: "b")).SequenceId; + return new Scene(editor, photo, held, target); + } + + private static Task StartIntoAsync(HttpClient client, byte[] data, Guid? target, bool restore = false) => + client.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(target, "IMG_0001.CR3", data.Length, Uploads.Sha(data), Shot, null, restore, IntoTarget: true), + ProtocolJson.Default.StartUploadRequest); + + private static async Task UsedAsync(HttpClient client) + { + using var me = await client.GetAsync("/api/v1/auth/me"); + return (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes; + } + + private static async Task> FramesOfAsync(TestServer server, Guid sequenceId, byte[] photo) => + [.. (await server.Database.Frames.FindBySha256Async(Uploads.Sha(photo))).Where(f => f.SequenceId == sequenceId)]; + + private sealed record Scene(HttpClient Editor, byte[] Photo, CompleteUploadResponse Held, Guid Target); + + /// The fake ingest, with a step that runs once just before the next commit (another writer filing the same photo). + private sealed class RacingIngest(FakeArchiveIngest inner) : IArchiveIngest + { + public FakeArchiveIngest Inner => inner; + + public Func? Before { get; set; } + + public async Task CommitAsync(IngestRequest request, CancellationToken cancellationToken = default) + { + if (Before is { } before) + { + Before = null; + await before(request); + } + return await inner.CommitAsync(request, cancellationToken); + } + } +}