feat(server): photo lookup endpoint and uploads into a target (owner review 6, P2)
Clients must recognise the same time-lapse filed under another id on the server, and an integrated pair must get a photo into the server sequence even when another sequence there already holds it. The server now answers both, authorised and scoped by itself. - POST /api/v1/photos/lookup (Read right): per probe, the sequences of the served archive that hold its hashes in a frame, tombstoned them (deleted or replaced by a conversion), match its unhashed photos by size and capture millisecond, or carry its session key; only ids and counts, batched raw SQL, 413 too_many beyond 200 probes or 20,000 items, a 4 MB body limit so a full request fits. - Uploads with IntoTarget: sequenceId required, never with Restore, the target must be the served archive's and the caller its owner, an admin or an editor. Only a copy in the target stands in; a tombstone keeps the photo deleted. Otherwise the server copies an intact file of another sequence of the served archive (never another archive of the index) into the target: quota and room checked, verified copy, hash compared (a damaged source falls back to a normal upload), committed refusing removed photos, used bytes charged only for a new frame, frames event published, answered copied. - server/info announces photo-lookup and upload-into (the desktop's hosted share too). Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
283d04d9cd
commit
94a56278e2
8 files changed
+926
-7
No files matched your search
@@ -0,0 +1,46 @@
|
||||
using System.Globalization;
|
||||
using Microsoft.AspNetCore.Http.Metadata;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Server.Security;
|
||||
using Chronos.Server.Storage;
|
||||
|
||||
namespace Chronos.Server.Api;
|
||||
|
||||
/// <summary>
|
||||
/// <c>POST /api/v1/photos/lookup</c> (owner review 6, #73; capability <see cref="ServerCapabilities.PhotoLookup"/>): which
|
||||
/// sequences of the served archive hold photos a client sampled from its own sequences (<see cref="PhotoLookup"/>). Needs
|
||||
/// the Read right; at most <see cref="PhotoLookupRequest.MaxProbes"/> probes and <see cref="PhotoLookupRequest.MaxItems"/>
|
||||
/// items, 413 <c>too_many</c> beyond.
|
||||
/// </summary>
|
||||
public static class PhotoLookupEndpoints
|
||||
{
|
||||
/// <summary>
|
||||
/// The body a full request may take: 20,000 items of about 70 bytes each are beyond the small limit of the other JSON
|
||||
/// endpoints (<see cref="Hosting.ServerHost.JsonBodyLimit"/>).
|
||||
/// </summary>
|
||||
private const long MaxBodyBytes = 4L * 1024 * 1024;
|
||||
|
||||
public static void Map(RouteGroupBuilder api)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(api);
|
||||
api.MapPost("/photos/lookup", LookupAsync).Requires(Permissions.Read).WithMetadata(new LookupBodyLimit());
|
||||
}
|
||||
|
||||
private static async Task<IResult> LookupAsync(HttpContext context, PhotoLookupRequest request, PhotoLookup lookup)
|
||||
{
|
||||
if (request.Probes is null) return ApiErrors.BadRequest(context, "bad_request", "probes");
|
||||
if (PhotoLookup.TooMany(request))
|
||||
{
|
||||
return ApiErrors.Result(context, StatusCodes.Status413PayloadTooLarge, "too_many",
|
||||
PhotoLookupRequest.MaxItems.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
return Results.Json(await lookup.LookupAsync(request, context.RequestAborted), ProtocolJson.Default.PhotoLookupResponse);
|
||||
}
|
||||
|
||||
/// <summary>The lookup's body limit, applied by routing as it selects the endpoint (as the chunk route's is).</summary>
|
||||
private sealed class LookupBodyLimit : IRequestSizeLimitMetadata
|
||||
{
|
||||
public long? MaxRequestBodySize => MaxBodyBytes;
|
||||
}
|
||||
}
|
||||
@@ -125,7 +125,8 @@ public sealed class ServerState
|
||||
public string Database { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>What this server offers clients (<see cref="ServerInfo.Capabilities"/>; <see cref="ServerCapabilities"/> names those clients act on).</summary>
|
||||
public List<string> Capabilities { get; } = ["uploads", "sync", "jobs", "render-nodes", "events", ServerCapabilities.Projects, ServerCapabilities.Videos];
|
||||
public List<string> Capabilities { get; } = ["uploads", "sync", "jobs", "render-nodes", "events", ServerCapabilities.Projects, ServerCapabilities.Videos,
|
||||
ServerCapabilities.PhotoLookup, ServerCapabilities.UploadInto];
|
||||
|
||||
public DateTimeOffset StartedAt { get; } = DateTimeOffset.UtcNow;
|
||||
}
|
||||
@@ -245,6 +245,7 @@ public static class ServerHost
|
||||
services.AddSingleton<ServerSettingsStore>();
|
||||
services.AddSingleton<UploadStore>();
|
||||
services.AddSingleton<UploadService>();
|
||||
services.AddSingleton<PhotoLookup>();
|
||||
services.AddSingleton<RenderStore>();
|
||||
services.AddSingleton<RenderUploads>();
|
||||
services.AddSingleton<SequenceJobs>();
|
||||
@@ -311,6 +312,7 @@ public static class ServerHost
|
||||
AuthEndpoints.Map(api);
|
||||
UserEndpoints.Map(api);
|
||||
SequenceEndpoints.Map(api);
|
||||
PhotoLookupEndpoints.Map(api);
|
||||
ProjectEndpoints.Map(api);
|
||||
LibraryEndpoints.Map(api);
|
||||
FileEndpoints.Map(api);
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using Chronos.Data;
|
||||
using Chronos.Protocol;
|
||||
|
||||
namespace Chronos.Server.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// <c>POST /api/v1/photos/lookup</c> (owner review 6, #73): which sequences of the served archive hold the photos a client
|
||||
/// sampled from its own sequences, so it recognises the same time-lapse filed under another id. Each probe's hashes are
|
||||
/// looked up in the frames and the tombstones, its unhashed photos by size and capture time to the millisecond, and its
|
||||
/// back-reference key in the session keys. The answer names only sequence ids and counts, never files or titles; another
|
||||
/// archive of the index is never looked at.
|
||||
/// </summary>
|
||||
public sealed class PhotoLookup(ChronosDatabase db, ServerArchive archive)
|
||||
{
|
||||
/// <summary>Parameters of one statement: the archive id and the values of one batch.</summary>
|
||||
private const int BatchParameters = 500;
|
||||
|
||||
private const int BatchValues = BatchParameters - 1;
|
||||
|
||||
/// <summary>The unhashed range query binds the archive and the two ends of the range besides the sizes.</summary>
|
||||
private const int UnhashedBatchValues = BatchParameters - 3;
|
||||
|
||||
/// <summary>
|
||||
/// The request goes beyond <see cref="PhotoLookupRequest.MaxProbes"/> probes or <see cref="PhotoLookupRequest.MaxItems"/>
|
||||
/// items (413 <c>too_many</c>).
|
||||
/// </summary>
|
||||
public static bool TooMany(PhotoLookupRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
var probes = request.Probes ?? [];
|
||||
if (probes.Count > PhotoLookupRequest.MaxProbes) return true;
|
||||
long items = 0;
|
||||
foreach (var probe in probes)
|
||||
{
|
||||
if (probe is null) continue;
|
||||
items += (probe.Sha256?.Count ?? 0) + (probe.Unhashed?.Count ?? 0) + (probe.SessionKey is null ? 0 : 1);
|
||||
}
|
||||
return items > PhotoLookupRequest.MaxItems;
|
||||
}
|
||||
|
||||
/// <summary>One result per probe, in the request's order; a probe that hit nothing has no hits.</summary>
|
||||
public async Task<PhotoLookupResponse> LookupAsync(PhotoLookupRequest request, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
var probes = (request.Probes ?? []).Select(Normalize).ToList();
|
||||
var hashes = probes.SelectMany(p => p.Hashes).Distinct(StringComparer.Ordinal).ToList();
|
||||
var keys = probes.SelectMany(p => p.Unhashed).Distinct().ToList();
|
||||
var sessions = probes.Where(p => p.SessionKey is not null).Select(p => p.SessionKey!).Distinct(StringComparer.Ordinal).ToList();
|
||||
|
||||
var held = new Dictionary<string, HashSet<Guid>>(StringComparer.Ordinal);
|
||||
var removed = new Dictionary<string, HashSet<Guid>>(StringComparer.Ordinal);
|
||||
var byKey = new Dictionary<PhotoKey, HashSet<Guid>>();
|
||||
var bySession = new Dictionary<string, HashSet<Guid>>(StringComparer.Ordinal);
|
||||
|
||||
var archiveId = archive.Id;
|
||||
await using (var session = await db.Database.OpenAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var batch in hashes.Chunk(BatchValues))
|
||||
{
|
||||
var (list, parameters) = InList(archiveId, batch);
|
||||
await foreach (var row in session.QueryAsync(
|
||||
"SELECT f.sequence_id, f.sha256 FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
|
||||
$"WHERE s.archive_id = $1 AND f.sha256 IN ({list})", parameters, ct).ConfigureAwait(false))
|
||||
Add(held, row.GetString(1), row.GetGuid(0));
|
||||
}
|
||||
var wanted = hashes.ToHashSet(StringComparer.Ordinal);
|
||||
foreach (var batch in hashes.Chunk(BatchValues))
|
||||
{
|
||||
var (list, parameters) = InList(archiveId, batch);
|
||||
await foreach (var row in session.QueryAsync(
|
||||
"SELECT r.sequence_id, r.sha256, r.replacement_sha256 FROM frame_removals r JOIN sequences s ON s.id = r.sequence_id " +
|
||||
$"WHERE s.archive_id = $1 AND (r.sha256 IN ({list}) OR r.replacement_sha256 IN ({list}))", parameters, ct).ConfigureAwait(false))
|
||||
{
|
||||
var sequence = row.GetGuid(0);
|
||||
if (wanted.Contains(row.GetString(1))) Add(removed, row.GetString(1), sequence);
|
||||
if (row.GetStringOrNull(2) is { } replacement && wanted.Contains(replacement)) Add(removed, replacement, sequence);
|
||||
}
|
||||
}
|
||||
// Sorted by capture time, so each batch asks for a narrow range of the frames.
|
||||
foreach (var batch in keys.OrderBy(k => k.CaptureTicks).Chunk(UnhashedBatchValues))
|
||||
{
|
||||
var wantedKeys = batch.ToHashSet();
|
||||
var parameters = new List<DbParam>
|
||||
{
|
||||
archiveId,
|
||||
new(new DateTime(batch[0].CaptureTicks, DateTimeKind.Unspecified), DbValueType.LocalDateTime),
|
||||
new(new DateTime(batch[^1].CaptureTicks + TimeSpan.TicksPerMillisecond - 1, DateTimeKind.Unspecified), DbValueType.LocalDateTime),
|
||||
};
|
||||
var sizes = new StringBuilder();
|
||||
foreach (var size in batch.Select(k => k.Size).Distinct())
|
||||
{
|
||||
parameters.Add(size);
|
||||
if (sizes.Length > 0) sizes.Append(", ");
|
||||
sizes.Append('$').Append(parameters.Count.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
await foreach (var row in session.QueryAsync(
|
||||
"SELECT f.sequence_id, f.size, f.captured_at FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
|
||||
$"WHERE s.archive_id = $1 AND f.captured_at >= $2 AND f.captured_at <= $3 AND f.size IN ({sizes})",
|
||||
[.. parameters], ct).ConfigureAwait(false))
|
||||
{
|
||||
if (row.GetDateTimeOrNull(2) is not { } captured) continue;
|
||||
var key = new PhotoKey(row.GetInt64(1), Millisecond(captured));
|
||||
if (wantedKeys.Contains(key)) Add(byKey, key, row.GetGuid(0));
|
||||
}
|
||||
}
|
||||
foreach (var batch in sessions.Chunk(BatchValues))
|
||||
{
|
||||
var (list, parameters) = InList(archiveId, batch);
|
||||
await foreach (var row in session.QueryAsync(
|
||||
$"SELECT session_key, sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key IN ({list})",
|
||||
parameters, ct).ConfigureAwait(false))
|
||||
Add(bySession, row.GetString(0), row.GetGuid(1));
|
||||
}
|
||||
}
|
||||
|
||||
var results = new List<ProbeResult>(probes.Count);
|
||||
foreach (var probe in probes)
|
||||
{
|
||||
var counts = new Dictionary<Guid, Counts>();
|
||||
foreach (var hash in probe.Hashes)
|
||||
{
|
||||
var holders = held.GetValueOrDefault(hash);
|
||||
foreach (var sequence in holders ?? []) Get(counts, sequence).Hashes++;
|
||||
// A tombstone of content the sequence still holds in a frame is stale: the photo counts once, as held.
|
||||
foreach (var sequence in removed.GetValueOrDefault(hash) ?? [])
|
||||
{
|
||||
if (holders?.Contains(sequence) != true) Get(counts, sequence).Removed++;
|
||||
}
|
||||
}
|
||||
foreach (var key in probe.Unhashed)
|
||||
{
|
||||
foreach (var sequence in byKey.GetValueOrDefault(key) ?? []) Get(counts, sequence).Unhashed++;
|
||||
}
|
||||
if (probe.SessionKey is { } sessionKey)
|
||||
{
|
||||
foreach (var sequence in bySession.GetValueOrDefault(sessionKey) ?? []) Get(counts, sequence).SessionKey = true;
|
||||
}
|
||||
results.Add(new ProbeResult(probe.Key, [.. counts
|
||||
.OrderByDescending(c => (c.Value.SessionKey ? 1 : 0, c.Value.Hashes + c.Value.Removed + c.Value.Unhashed))
|
||||
.ThenBy(c => c.Key)
|
||||
.Select(c => new PhotoLookupHit(c.Key, c.Value.Hashes, c.Value.Removed, c.Value.Unhashed, c.Value.SessionKey))]));
|
||||
}
|
||||
return new PhotoLookupResponse(results);
|
||||
}
|
||||
|
||||
/// <summary>A probe as a set: lower-case hashes, keys to the millisecond, a trimmed session key; each counted once.</summary>
|
||||
private static Probe Normalize(PhotoProbe? probe) =>
|
||||
probe is null
|
||||
? new Probe(string.Empty, [], [], null)
|
||||
: new Probe(
|
||||
probe.Key ?? string.Empty,
|
||||
[.. (probe.Sha256 ?? []).Where(h => h is { Length: 64 }).Select(h => h.ToLowerInvariant()).Distinct(StringComparer.Ordinal)],
|
||||
[.. (probe.Unhashed ?? []).Where(k => k is not null).Select(k => new PhotoKey(k.Size, Millisecond(k.CapturedAt))).Distinct()],
|
||||
string.IsNullOrWhiteSpace(probe.SessionKey) ? null : probe.SessionKey.Trim());
|
||||
|
||||
/// <summary>
|
||||
/// A capture time to the millisecond, as a wall-clock time whatever kind it came with (as
|
||||
/// <c>SequenceRepository.Wall</c> stores it): the index keeps microseconds, a client may send ticks.
|
||||
/// </summary>
|
||||
private static long Millisecond(DateTime value) => value.Ticks - value.Ticks % TimeSpan.TicksPerMillisecond;
|
||||
|
||||
/// <summary><c>$2, $3, …</c> for <paramref name="values"/>, after the archive id as <c>$1</c>.</summary>
|
||||
private static (string List, DbParam[] Parameters) InList(Guid archiveId, string[] values)
|
||||
{
|
||||
var parameters = new DbParam[values.Length + 1];
|
||||
parameters[0] = archiveId;
|
||||
var list = new StringBuilder();
|
||||
for (var i = 0; i < values.Length; i++)
|
||||
{
|
||||
parameters[i + 1] = values[i];
|
||||
if (i > 0) list.Append(", ");
|
||||
list.Append('$').Append((i + 2).ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
return (list.ToString(), parameters);
|
||||
}
|
||||
|
||||
private static void Add<TKey>(Dictionary<TKey, HashSet<Guid>> map, TKey key, Guid sequence) where TKey : notnull
|
||||
{
|
||||
if (!map.TryGetValue(key, out var set)) map[key] = set = [];
|
||||
set.Add(sequence);
|
||||
}
|
||||
|
||||
private static Counts Get(Dictionary<Guid, Counts> counts, Guid sequence)
|
||||
{
|
||||
if (!counts.TryGetValue(sequence, out var found)) counts[sequence] = found = new Counts();
|
||||
return found;
|
||||
}
|
||||
|
||||
private readonly record struct PhotoKey(long Size, long CaptureTicks);
|
||||
|
||||
private sealed record Probe(string Key, IReadOnlyList<string> Hashes, IReadOnlyList<PhotoKey> Unhashed, string? SessionKey);
|
||||
|
||||
private sealed class Counts
|
||||
{
|
||||
public int Hashes { get; set; }
|
||||
|
||||
public int Removed { get; set; }
|
||||
|
||||
public int Unhashed { get; set; }
|
||||
|
||||
public bool SessionKey { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -47,13 +47,31 @@ public static class UploadRules
|
||||
/// <summary>
|
||||
/// The frame that stands for an upload of content the archive already has (<paramref name="copies"/>): an intact copy,
|
||||
/// the one in the upload's <paramref name="target"/> sequence first. Null when there is none, or when the target holds
|
||||
/// the content only in damaged copies: the upload's bytes repair them, whatever other sequences hold.
|
||||
/// the content only in damaged copies: the upload's bytes repair them, whatever other sequences hold. With
|
||||
/// <paramref name="intoTarget"/> (<c>StartUploadRequest.IntoTarget</c>, owner review 6, #73) only an intact copy in the
|
||||
/// target stands in: the photo is to be filed there, a copy elsewhere is no answer.
|
||||
/// </summary>
|
||||
public static Frame? StandIn(string archiveRoot, IReadOnlyList<Frame> copies, Guid? target)
|
||||
public static Frame? StandIn(string archiveRoot, IReadOnlyList<Frame> copies, Guid? target, bool intoTarget = false)
|
||||
{
|
||||
var intact = copies.Where(f => IsIntact(archiveRoot, f)).ToList();
|
||||
if (copies.Any(f => f.SequenceId == target) && !intact.Any(f => f.SequenceId == target)) return null;
|
||||
return intact.FirstOrDefault(f => f.SequenceId == target) ?? intact.FirstOrDefault();
|
||||
return intact.FirstOrDefault(f => f.SequenceId == target) ?? (intoTarget ? null : intact.FirstOrDefault());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The file the server copies into <paramref name="target"/> for an upload into it (owner review 6, #73): an intact copy
|
||||
/// held by another sequence of the served archive <paramref name="archiveId"/>, resolved under that archive's
|
||||
/// <paramref name="archiveRoot"/>. A frame of a sequence of another archive of the index (<paramref name="archiveOf"/>
|
||||
/// names each copy's sequence's archive; a desktop sharing its library keeps other archives' frames in the same index)
|
||||
/// is never a source. Null when there is none.
|
||||
/// </summary>
|
||||
public static Frame? CopySource(string archiveRoot, IReadOnlyList<Frame> copies, Guid target, Guid archiveId,
|
||||
IReadOnlyDictionary<Guid, Guid> archiveOf)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(copies);
|
||||
ArgumentNullException.ThrowIfNull(archiveOf);
|
||||
return copies.FirstOrDefault(f => f.SequenceId != target && archiveOf.TryGetValue(f.SequenceId, out var archive) && archive == archiveId
|
||||
&& IsIntact(archiveRoot, f));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -2,6 +2,8 @@ using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using Microsoft.AspNetCore.Http.Features;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Import;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Repositories;
|
||||
@@ -56,7 +58,18 @@ public sealed class UploadService(
|
||||
if (request.Size <= 0) return Fail<StartUploadResponse>(400, "bad_request", "size");
|
||||
if (request.Size > options.MaxUploadBytes) return Fail<StartUploadResponse>(413, "too_large", options.MaxUploadBytes.ToString(CultureInfo.InvariantCulture));
|
||||
if (request.SessionKey is { Length: > 200 }) return Fail<StartUploadResponse>(400, "bad_request", "sessionKey");
|
||||
if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null)
|
||||
if (request.IntoTarget)
|
||||
{
|
||||
// Authorised here, never trusted to the client (owner review 6, #73): the served archive's sequence, which the
|
||||
// caller owns or may edit, and never a tombstone dropped through a copy the server makes itself.
|
||||
if (request.SequenceId is not { } into) return Fail<StartUploadResponse>(400, "bad_request", "sequenceId");
|
||||
if (request.Restore) return Fail<StartUploadResponse>(400, "bad_request", "restore");
|
||||
if (await ServedItems.SequenceAsync(db, archive, into, cancellationToken) is not { } intoSequence)
|
||||
return Fail<StartUploadResponse>(404, "not_found", "sequence");
|
||||
if (intoSequence.OwnerUserId != caller.Id && !caller.IsAdmin && !caller.Has(Permissions.Edit))
|
||||
return Fail<StartUploadResponse>(403, "forbidden");
|
||||
}
|
||||
else if (request.SequenceId is { } requested && await db.Sequences.GetAsync(requested, cancellationToken) is null)
|
||||
return Fail<StartUploadResponse>(404, "not_found", "sequence");
|
||||
|
||||
var sha = request.Sha256.ToLowerInvariant();
|
||||
@@ -70,9 +83,16 @@ public sealed class UploadService(
|
||||
var removed = await UploadRules.RemovedFromTargetAsync(db, sha, target, copies, cancellationToken);
|
||||
if (removed && !request.Restore) return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, null);
|
||||
// Only a copy that is still whole stands for the upload: a client may delete its file on this answer. A photo
|
||||
// brought back is filed into its sequence, whatever other sequences hold it.
|
||||
if (UploadRules.StandIn(archive.Root, removed ? [] : copies, target) is { } known)
|
||||
// brought back is filed into its sequence, whatever other sequences hold it. An upload into the target hears only of
|
||||
// a copy there.
|
||||
if (UploadRules.StandIn(archive.Root, removed ? [] : copies, target, request.IntoTarget) is { } known)
|
||||
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, known.Id);
|
||||
// Into a target that holds no copy, while another sequence of the served archive holds an intact one: the server
|
||||
// files its own file there, no bytes on the wire. A copy that does not read back as the request's content leaves
|
||||
// the client to send the bytes.
|
||||
if (request.IntoTarget && target is { } targetId && !copies.Any(f => f.SequenceId == targetId)
|
||||
&& await CopyIntoAsync(caller, sha, targetId, copies, cancellationToken) is { } copied)
|
||||
return copied;
|
||||
|
||||
var resumable = await db.Uploads.FindReceivingAsync(caller.Id, sha, cancellationToken);
|
||||
if (resumable is not null && resumable.Size == request.Size && resumable.SequenceId == request.SequenceId && store.Exists(resumable.Id))
|
||||
@@ -325,6 +345,96 @@ public sealed class UploadService(
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Files into <paramref name="target"/> the server's own file of <paramref name="sha"/> held by another sequence of the
|
||||
/// served archive (<see cref="StartUploadRequest.IntoTarget"/>, owner review 6, #73): checked against the caller's quota
|
||||
/// and the archive's room, copied to staging and read back (<see cref="VerifiedCopy"/>), its SHA-256 compared with the
|
||||
/// request's, and committed as an upload of it would be, refusing a photo the target deleted. Used bytes grow only when
|
||||
/// a new frame was filed. Null when there is no such file, or when the copy is not the request's content (a damaged
|
||||
/// source): the client then sends the bytes.
|
||||
/// </summary>
|
||||
private async Task<Outcome<StartUploadResponse>?> CopyIntoAsync(Caller caller, string sha, Guid target, IReadOnlyList<Frame> copies,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var archiveOf = new Dictionary<Guid, Guid>();
|
||||
foreach (var sequenceId in copies.Select(f => f.SequenceId).Distinct())
|
||||
{
|
||||
if (await db.Sequences.GetAsync(sequenceId, cancellationToken) is { } sequence) archiveOf[sequenceId] = sequence.ArchiveId;
|
||||
}
|
||||
if (UploadRules.CopySource(archive.Root, copies, target, archive.Id, archiveOf) is not { } source
|
||||
|| ArchivePathGuard.ExistingFile(archive.Root, source.RelativePath) is not { } file)
|
||||
return null;
|
||||
|
||||
if (caller.User.QuotaBytes > 0)
|
||||
{
|
||||
var pending = await db.Uploads.PendingBytesAsync(caller.Id, cancellationToken);
|
||||
if (caller.User.UsedBytes + pending + source.Size > caller.User.QuotaBytes)
|
||||
return Fail<StartUploadResponse>(403, "quota_exceeded", caller.User.QuotaBytes.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
if (volumes.Measure(archive.Root) is { } volume && volume.FreeBytes < source.Size + DiskReserve)
|
||||
return Fail<StartUploadResponse>(507, "insufficient_storage");
|
||||
|
||||
// Runs to the end once started, even if the client disconnects, as a completion does.
|
||||
var stopping = lifetime.ApplicationStopping;
|
||||
var staged = Path.Combine(options.StagingPath, Guid.CreateVersion7().ToString("N") + Path.GetExtension(source.FileName).ToLowerInvariant());
|
||||
try
|
||||
{
|
||||
var copied = await VerifiedCopy.CopyAsync(file.FullName, staged, verify: true, cancellationToken: stopping);
|
||||
if (copied != sha)
|
||||
{
|
||||
logger.LogWarning("Frame {Frame} ({File}) holds SHA-256 {Actual} instead of {Expected}: not copied into sequence {Target}, the client sends the photo",
|
||||
source.Id, source.RelativePath, copied, sha, target);
|
||||
UploadStore.TryDelete(staged);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
logger.LogWarning(ex, "Frame {Frame} ({File}) could not be copied into sequence {Target}: the client sends the photo", source.Id, source.RelativePath, target);
|
||||
UploadStore.TryDelete(staged);
|
||||
return null;
|
||||
}
|
||||
|
||||
IngestResult result;
|
||||
try
|
||||
{
|
||||
result = await ingest.CommitAsync(new IngestRequest(archive.Id, target, NewSequenceTitle: null, SessionKey: null, staged,
|
||||
source.OriginalFileName ?? source.FileName, source.Size, sha, source.CapturedAt, caller.Id, RefuseRemoved: true), stopping);
|
||||
}
|
||||
catch (PhotoRemovedException)
|
||||
{
|
||||
// Deleted from the target since the start checked: it stays deleted.
|
||||
UploadStore.TryDelete(staged);
|
||||
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, null);
|
||||
}
|
||||
catch (Exception ex) when (ex is LeaseUnavailableException or ArchiveOfflineException or SequenceFolderMovedException or FolderMovedException)
|
||||
{
|
||||
logger.LogWarning("Frame {Frame} not copied into sequence {Target} for now: {Reason}", source.Id, target, ex.Message);
|
||||
UploadStore.TryDelete(staged);
|
||||
return Fail<StartUploadResponse>(503, ex switch
|
||||
{
|
||||
ArchiveOfflineException => "archive_offline",
|
||||
SequenceFolderMovedException or FolderMovedException => "folder_moved",
|
||||
_ => "sequence_busy",
|
||||
});
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
logger.LogError(ex, "Copying frame {Frame} into sequence {Target} failed", source.Id, target);
|
||||
UploadStore.TryDelete(staged);
|
||||
return Fail<StartUploadResponse>(503, "commit_failed");
|
||||
}
|
||||
|
||||
if (File.Exists(staged)) UploadStore.TryDelete(staged);
|
||||
if (!result.Duplicate && !result.Repaired) await db.Users.AddUsedBytesAsync(caller.Id, source.Size, CancellationToken.None);
|
||||
if (await db.Sequences.GetAsync(result.SequenceId, CancellationToken.None) is { } filed)
|
||||
{
|
||||
await sequenceJobs.FramesAddedAsync(filed, CancellationToken.None);
|
||||
events.Publish(new SequenceChangedEvent(filed.Id, "frames", filed.FramesVersion, filed.SettingsVersion));
|
||||
}
|
||||
return new StartUploadResponse(null, options.ChunkSizeBytes, [], AlreadyExists: true, result.FrameId, Copied: true);
|
||||
}
|
||||
|
||||
private async Task FailAsync(Upload upload)
|
||||
{
|
||||
await db.Uploads.TryTransitionAsync(upload.Id, UploadState.Committing, UploadState.Failed, cancellationToken: CancellationToken.None);
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Server.Configuration;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Server.Tests.Api;
|
||||
|
||||
/// <summary><c>POST /api/v1/photos/lookup</c> (owner review 6, #73): counts per probe, scoped to the served archive.</summary>
|
||||
[Category("server")]
|
||||
public sealed class PhotoLookupTests(TestContext context)
|
||||
{
|
||||
private static readonly DateTime Shot = new DateTime(2026, 9, 28, 18, 2, 11, 123, DateTimeKind.Unspecified).AddTicks(4560);
|
||||
|
||||
[Test]
|
||||
public async Task EachProbeCountsTheSequencesThatHoldItsPhotos()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
await CountsAsync(server, context);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[RequiresEnvironment("CHRONOS_TEST_PG")]
|
||||
public async Task EachProbeCountsTheSequencesThatHoldItsPhotosOnPostgres()
|
||||
{
|
||||
await using var schema = await PgSchema.CreateAsync(context);
|
||||
await using var server = await schema.StartAsync(context);
|
||||
Assert.Equal(Dialect.Postgres, server.Database.Dialect);
|
||||
await CountsAsync(server, context);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OnlyTheServedArchiveIsLookedAt()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
await ScopedAsync(server, context);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[RequiresEnvironment("CHRONOS_TEST_PG")]
|
||||
public async Task OnlyTheServedArchiveIsLookedAtOnPostgres()
|
||||
{
|
||||
await using var schema = await PgSchema.CreateAsync(context);
|
||||
await using var server = await schema.StartAsync(context);
|
||||
await ScopedAsync(server, context);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheLookupNeedsTheReadRight()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
|
||||
using (var refused = await LookupAsync(uploader, new PhotoLookupRequest([new PhotoProbe("a", [Hash('a')])])))
|
||||
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
||||
using (var anonymous = await LookupAsync(server.Client, new PhotoLookupRequest([new PhotoProbe("a", [Hash('a')])])))
|
||||
Assert.Equal(HttpStatusCode.Unauthorized, anonymous.StatusCode);
|
||||
var (viewer, _, _) = await server.UserAsync("guest", "viewer");
|
||||
using var allowed = await LookupAsync(viewer, new PhotoLookupRequest([new PhotoProbe("a", [Hash('a')])]));
|
||||
Assert.Empty(Assert.NotNull(await allowed.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Single().Hits);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task MoreThan200ProbesOr20000ItemsAreRefused()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
|
||||
var most = Enumerable.Range(0, PhotoLookupRequest.MaxProbes).Select(i => new PhotoProbe("p" + i, [Hash('b')])).ToList();
|
||||
using (var ok = await LookupAsync(editor, new PhotoLookupRequest(most)))
|
||||
Assert.Equal(PhotoLookupRequest.MaxProbes, (await ok.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Count);
|
||||
using (var probes = await LookupAsync(editor, new PhotoLookupRequest([.. most, new PhotoProbe("one more", [Hash('b')])])))
|
||||
await probes.ExpectErrorAsync(HttpStatusCode.RequestEntityTooLarge, "too_many");
|
||||
|
||||
// 20,000 items fit in one request (a body larger than the other JSON calls take); one more does not.
|
||||
var hashes = Enumerable.Range(0, PhotoLookupRequest.MaxItems - 100).Select(i => i.ToString("x64", System.Globalization.CultureInfo.InvariantCulture)).ToList();
|
||||
var unhashed = Enumerable.Range(0, 99).Select(i => new PhotoKeyDto(1000 + i, Shot.AddSeconds(i))).ToList();
|
||||
var full = new PhotoProbe("full", hashes, unhashed, "chronos-sync:" + Guid.NewGuid().ToString("N"));
|
||||
Assert.Equal(PhotoLookupRequest.MaxItems, new PhotoLookupRequest([full]).CountItems());
|
||||
using (var ok = await LookupAsync(editor, new PhotoLookupRequest([full])))
|
||||
Assert.Empty((await ok.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Single().Hits);
|
||||
var over = full with { Unhashed = [.. unhashed, new PhotoKeyDto(5, Shot)] };
|
||||
using (var items = await LookupAsync(editor, new PhotoLookupRequest([over])))
|
||||
await items.ExpectErrorAsync(HttpStatusCode.RequestEntityTooLarge, "too_many");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheServerAnnouncesTheLookupAndUploadsIntoATarget()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
using var response = await server.Client.GetAsync("/api/v1/server/info");
|
||||
var info = await response.ReadAsync(ProtocolJson.Default.ServerInfo);
|
||||
Assert.Contains(ServerCapabilities.PhotoLookup, info.Capabilities);
|
||||
Assert.Contains(ServerCapabilities.UploadInto, info.Capabilities);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sequence A holds p1, p2 and p3 (p3 captured at <see cref="Shot"/>); B holds p4, tombstoned p5, replaced p6 by p7 and
|
||||
/// is filed under a session key. One probe counts both, another hits nothing.
|
||||
/// </summary>
|
||||
internal static async Task CountsAsync(TestServer server, TestContext context)
|
||||
{
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var photos = Enumerable.Range(1, 7).Select(i => Uploads.Photo(3000 + i, 700 + i)).ToArray();
|
||||
var a = (await Uploads.UploadAsync(editor, photos[0], "IMG_0001.JPG", sessionKey: "a")).SequenceId;
|
||||
await Uploads.UploadAsync(editor, photos[1], "IMG_0002.JPG", sessionKey: "a");
|
||||
await Uploads.UploadAsync(editor, photos[2], "IMG_0003.JPG", sessionKey: "a", capturedAt: Shot);
|
||||
var b = (await Uploads.UploadAsync(editor, photos[3], "IMG_0004.JPG", sessionKey: "b")).SequenceId;
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
await server.Database.FrameRemovals.SyncSequenceAsync(b,
|
||||
[
|
||||
new FrameRemoval(b, Uploads.Sha(photos[4]), null, RemovalReason.Deleted, now),
|
||||
new FrameRemoval(b, Uploads.Sha(photos[5]), Uploads.Sha(photos[6]), RemovalReason.Replaced, now),
|
||||
// A stale tombstone: B still holds p4, which counts once, as held.
|
||||
new FrameRemoval(b, Uploads.Sha(photos[3]), null, RemovalReason.Deleted, now),
|
||||
], context.Cancellation);
|
||||
var archive = (await server.Database.Sequences.GetAsync(b, context.Cancellation))!.ArchiveId;
|
||||
var key = "chronos-sync:" + Guid.NewGuid().ToString("N");
|
||||
await server.Database.Sequences.MapSessionKeyAsync(archive, key, b, context.Cancellation);
|
||||
|
||||
var probe = new PhotoProbe("local-1",
|
||||
[Uploads.Sha(photos[0]).ToUpperInvariant(), Uploads.Sha(photos[1]), Uploads.Sha(photos[1]), Uploads.Sha(photos[3]), Uploads.Sha(photos[4]),
|
||||
Uploads.Sha(photos[6]), Hash('c')],
|
||||
[
|
||||
// The same millisecond as p3 (the index keeps microseconds); the next millisecond, another size: no match.
|
||||
new PhotoKeyDto(photos[2].Length, new DateTime(2026, 9, 28, 18, 2, 11, 123).AddTicks(9999)),
|
||||
new PhotoKeyDto(photos[2].Length, new DateTime(2026, 9, 28, 18, 2, 11, 124)),
|
||||
new PhotoKeyDto(photos[2].Length + 1, Shot),
|
||||
],
|
||||
key);
|
||||
var nothing = new PhotoProbe("local-2", [Hash('d')], [new PhotoKeyDto(photos[2].Length, Shot.AddDays(1))], "chronos-sync:" + Guid.NewGuid().ToString("N"));
|
||||
using var response = await LookupAsync(editor, new PhotoLookupRequest([probe, nothing]));
|
||||
var results = (await response.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results;
|
||||
|
||||
Assert.SequenceEqual(["local-1", "local-2"], results.Select(r => r.Key));
|
||||
var hits = results[0].Hits.ToDictionary(h => h.SequenceId);
|
||||
Assert.Equal(2, hits.Count);
|
||||
Assert.Equal(new PhotoLookupHit(a, Hashes: 2, Removed: 0, Unhashed: 1, SessionKey: false), hits[a]);
|
||||
Assert.Equal(new PhotoLookupHit(b, Hashes: 1, Removed: 2, Unhashed: 0, SessionKey: true), hits[b]);
|
||||
Assert.Empty(results[1].Hits);
|
||||
}
|
||||
|
||||
/// <summary>A sequence of another archive of the index holding the same photo and session key is never answered.</summary>
|
||||
internal static async Task ScopedAsync(TestServer server, TestContext context)
|
||||
{
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor");
|
||||
var photo = Uploads.Photo(4000, 801);
|
||||
var mine = (await Uploads.UploadAsync(editor, photo, sessionKey: "mine", capturedAt: Shot)).SequenceId;
|
||||
var key = "chronos-sync:" + Guid.NewGuid().ToString("N");
|
||||
var foreign = await ForeignFrameAsync(server, context, photo, Shot, key);
|
||||
|
||||
using var response = await LookupAsync(editor, new PhotoLookupRequest([new PhotoProbe("x", [Uploads.Sha(photo)], [new PhotoKeyDto(photo.Length, Shot)], key)]));
|
||||
var hit = (await response.ReadAsync(ProtocolJson.Default.PhotoLookupResponse)).Results.Single().Hits.Single();
|
||||
Assert.Equal(new PhotoLookupHit(mine, 1, 0, 1, false), hit);
|
||||
Assert.NotEqual(foreign.SequenceId, hit.SequenceId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A frame of <paramref name="data"/> in a sequence of a second archive of the index (as a desktop sharing its library
|
||||
/// keeps other archives' sequences), its file under that archive's root; optionally filed under <paramref name="sessionKey"/>.
|
||||
/// </summary>
|
||||
internal static async Task<Frame> ForeignFrameAsync(TestServer server, TestContext context, byte[] data, DateTime? capturedAt = null, string? sessionKey = null)
|
||||
{
|
||||
var db = server.Database;
|
||||
var other = new Archive { Id = Guid.CreateVersion7(), Name = "Another archive", RootPath = context.CreateTempDirectory("other-archive"), Created = DateTimeOffset.UtcNow };
|
||||
await db.Archives.UpsertAsync(other, context.Cancellation);
|
||||
var sequence = new Sequence { Id = Guid.CreateVersion7(), ArchiveId = other.Id, Folder = "2026/other", UpdatedAt = DateTimeOffset.UtcNow };
|
||||
await db.Sequences.UpsertAsync(sequence, context.Cancellation);
|
||||
const string relative = "2026/other/originals/0000_IMG_0001.CR3";
|
||||
var path = Path.Combine(other.RootPath, relative);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
await File.WriteAllBytesAsync(path, data, context.Cancellation);
|
||||
var frame = new Frame
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
SequenceId = sequence.Id,
|
||||
Index = 0,
|
||||
RelativePath = relative,
|
||||
FileName = Path.GetFileName(relative),
|
||||
OriginalFileName = "IMG_0001.CR3",
|
||||
Size = data.Length,
|
||||
Sha256 = Uploads.Sha(data),
|
||||
CapturedAt = capturedAt,
|
||||
};
|
||||
await db.Frames.InsertAsync(frame, context.Cancellation);
|
||||
await db.Sequences.RefreshStatsAsync(sequence.Id, context.Cancellation);
|
||||
if (sessionKey is not null) await db.Sequences.MapSessionKeyAsync(other.Id, sessionKey, sequence.Id, context.Cancellation);
|
||||
return frame;
|
||||
}
|
||||
|
||||
internal static Task<HttpResponseMessage> LookupAsync(HttpClient client, PhotoLookupRequest request) =>
|
||||
client.PostAsJsonAsync("/api/v1/photos/lookup", request, ProtocolJson.Default.PhotoLookupRequest);
|
||||
|
||||
private static string Hash(char c) => new(c, 64);
|
||||
}
|
||||
|
||||
/// <summary>A throw-away schema on the PostgreSQL test server (<c>CHRONOS_TEST_PG</c>), selected through <c>Search Path</c>.</summary>
|
||||
internal sealed class PgSchema : IAsyncDisposable
|
||||
{
|
||||
private readonly IDatabase _admin;
|
||||
private readonly string _name;
|
||||
|
||||
private PgSchema(IDatabase admin, string name, string connectionString)
|
||||
{
|
||||
_admin = admin;
|
||||
_name = name;
|
||||
ConnectionString = connectionString;
|
||||
}
|
||||
|
||||
public string ConnectionString { get; }
|
||||
|
||||
public static async Task<PgSchema> CreateAsync(TestContext context)
|
||||
{
|
||||
var baseConnection = Environment.GetEnvironmentVariable("CHRONOS_TEST_PG")!;
|
||||
var admin = DatabaseFactory.Open(DatabaseOptions.Postgres(baseConnection));
|
||||
var name = "chronos_l_" + Guid.NewGuid().ToString("N")[..16];
|
||||
await using (var session = await admin.OpenAsync(context.Cancellation))
|
||||
await session.ExecuteScriptAsync($"CREATE SCHEMA {name}", context.Cancellation);
|
||||
return new PgSchema(admin, name, baseConnection.TrimEnd(';', ' ') + $";Search Path={name}");
|
||||
}
|
||||
|
||||
/// <summary>A test server on this schema.</summary>
|
||||
public Task<TestServer> StartAsync(TestContext context, Action<ServerOptions>? configure = null,
|
||||
Action<Microsoft.Extensions.DependencyInjection.IServiceCollection>? services = null) =>
|
||||
TestServer.StartAsync(context, o =>
|
||||
{
|
||||
o.Database = DatabaseKind.Postgres;
|
||||
o.DbConnectionString = ConnectionString;
|
||||
configure?.Invoke(o);
|
||||
}, services);
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await using (var session = await _admin.OpenAsync(CancellationToken.None))
|
||||
await session.ExecuteScriptAsync($"DROP SCHEMA IF EXISTS {_name} CASCADE", CancellationToken.None);
|
||||
await _admin.DisposeAsync();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
using System.Net;
|
||||
using System.Net.Http.Json;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Data;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Protocol;
|
||||
using Chronos.Server.Events;
|
||||
using Chronos.Server.Tests.Api;
|
||||
using Chronos.Testing;
|
||||
|
||||
namespace Chronos.Server.Tests.Storage;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="StartUploadRequest.IntoTarget"/> (owner review 6, #73): a photo another sequence of the served archive holds is
|
||||
/// copied into the target by the server itself, authorised and scoped on the server.
|
||||
/// </summary>
|
||||
[Category("server")]
|
||||
public sealed class UploadIntoTests(TestContext context)
|
||||
{
|
||||
private static readonly DateTime Shot = new(2026, 9, 28, 18, 2, 11, 250, DateTimeKind.Unspecified);
|
||||
|
||||
[Test]
|
||||
public async Task APhotoHeldElsewhereIsCopiedIntoTheTargetAndChargedOnce()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
await CopiedAsync(server, context);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[RequiresEnvironment("CHRONOS_TEST_PG")]
|
||||
public async Task APhotoHeldElsewhereIsCopiedIntoTheTargetAndChargedOnceOnPostgres()
|
||||
{
|
||||
await using var schema = await PgSchema.CreateAsync(context);
|
||||
await using var server = await schema.StartAsync(context);
|
||||
Assert.Equal(Dialect.Postgres, server.Database.Dialect);
|
||||
await CopiedAsync(server, context);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ACommitThatFindsThePhotoFiledMeanwhileChargesNothing()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context, services: s =>
|
||||
{
|
||||
s.AddSingleton<RacingIngest>();
|
||||
s.AddSingleton<IArchiveIngest>(sp => sp.GetRequiredService<RacingIngest>());
|
||||
});
|
||||
var scene = await SceneAsync(server);
|
||||
var racing = server.Services.GetRequiredService<RacingIngest>();
|
||||
Guid? raced = null;
|
||||
// Another writer files the same photo into the target just before this commit: the ingest answers a duplicate.
|
||||
racing.Before = async request =>
|
||||
{
|
||||
var copy = request.StagedPath + ".race";
|
||||
File.Copy(request.StagedPath, copy);
|
||||
raced = (await racing.Inner.CommitAsync(request with { StagedPath = copy }, context.Cancellation)).FrameId;
|
||||
};
|
||||
var used = await UsedAsync(scene.Editor);
|
||||
|
||||
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target);
|
||||
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.True(answer.AlreadyExists && answer.Copied);
|
||||
Assert.Equal(Assert.NotNull(raced), answer.FrameId);
|
||||
Assert.Equal(used, await UsedAsync(scene.Editor));
|
||||
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TheCopyIsRefusedBeyondTheQuota()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
// The two uploads of the scene take 9,000 bytes; copying the 5,000-byte photo would take the account beyond 10,000.
|
||||
var scene = await SceneAsync(server, quota: 10_000);
|
||||
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target);
|
||||
await response.ExpectErrorAsync(HttpStatusCode.Forbidden, "quota_exceeded");
|
||||
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OnlyTheOwnerAnAdminOrAnEditorMayUploadIntoATarget()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
|
||||
|
||||
// An upload-only account may not file into another account's sequence.
|
||||
using (var refused = await StartIntoAsync(uploader, scene.Photo, scene.Target))
|
||||
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
||||
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
||||
|
||||
// Into its own sequence it may: the server copies the photo the editor's sequence holds.
|
||||
var own = (await Uploads.UploadAsync(uploader, Uploads.Photo(3000, 903), "IMG_0200.CR3", sessionKey: "roof")).SequenceId;
|
||||
using (var allowed = await StartIntoAsync(uploader, scene.Photo, own))
|
||||
Assert.True((await allowed.ReadAsync(ProtocolJson.Default.StartUploadResponse)).Copied);
|
||||
Assert.Count(1, await FramesOfAsync(server, own, scene.Photo));
|
||||
|
||||
// An administrator may.
|
||||
var admin = await server.AdminAsync();
|
||||
using (var byAdmin = await StartIntoAsync(admin, scene.Photo, scene.Target))
|
||||
Assert.True((await byAdmin.ReadAsync(ProtocolJson.Default.StartUploadResponse)).Copied);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ATargetOfAnotherArchiveIsUnknown()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
var foreign = await PhotoLookupTests.ForeignFrameAsync(server, context, Uploads.Photo(2000, 904));
|
||||
using (var other = await StartIntoAsync(scene.Editor, scene.Photo, foreign.SequenceId))
|
||||
Assert.Equal("sequence", (await other.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found")).Details);
|
||||
using (var unknown = await StartIntoAsync(scene.Editor, scene.Photo, Guid.CreateVersion7()))
|
||||
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
|
||||
Assert.Empty(await FramesOfAsync(server, foreign.SequenceId, scene.Photo));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnUploadIntoATargetNamesItAndNeverRestores()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
using (var restore = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target, restore: true))
|
||||
Assert.Equal("restore", (await restore.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
||||
using (var none = await StartIntoAsync(scene.Editor, scene.Photo, null))
|
||||
Assert.Equal("sequenceId", (await none.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ATargetThatHoldsThePhotoAnswersItsFrame()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Held.SequenceId);
|
||||
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
||||
Assert.True(answer.AlreadyExists);
|
||||
Assert.False(answer.Copied);
|
||||
Assert.Equal(scene.Held.FrameId, answer.FrameId);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APhotoTheTargetDeletedStaysDeleted()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
await server.Database.FrameRemovals.SyncSequenceAsync(scene.Target,
|
||||
[new FrameRemoval(scene.Target, Uploads.Sha(scene.Photo), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], context.Cancellation);
|
||||
var requests = server.Services.GetRequiredService<FakeArchiveIngest>().Requests.Count;
|
||||
|
||||
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target);
|
||||
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
||||
Assert.True(answer.AlreadyExists);
|
||||
Assert.Null(answer.FrameId);
|
||||
Assert.False(answer.Copied);
|
||||
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
||||
Assert.Equal(requests, server.Services.GetRequiredService<FakeArchiveIngest>().Requests.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APhotoOnlyAnotherArchiveHoldsIsUploaded()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
var elsewhere = Uploads.Photo(2500, 905);
|
||||
await PhotoLookupTests.ForeignFrameAsync(server, context, elsewhere);
|
||||
|
||||
using var response = await StartIntoAsync(scene.Editor, elsewhere, scene.Target);
|
||||
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
|
||||
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
||||
Assert.NotNull(answer.UploadId);
|
||||
Assert.False(answer.AlreadyExists || answer.Copied);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ADamagedSourceLeavesTheClientToSendThePhoto()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
// The file keeps its size, so the frame reads as intact; its bytes are no longer the photo.
|
||||
var source = Assert.NotNull(await server.Database.Frames.GetAsync(scene.Held.FrameId));
|
||||
await File.WriteAllBytesAsync(Path.Combine(server.Options.ArchivePath, source.RelativePath), Uploads.Photo(scene.Photo.Length, 906), context.Cancellation);
|
||||
|
||||
using (var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target))
|
||||
{
|
||||
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
|
||||
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
||||
Assert.NotNull(answer.UploadId);
|
||||
Assert.False(answer.Copied);
|
||||
}
|
||||
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
|
||||
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task WithoutIntoTargetACopyElsewhereStillStandsIn()
|
||||
{
|
||||
await using var server = await TestServer.StartAsync(context);
|
||||
var scene = await SceneAsync(server);
|
||||
var used = await UsedAsync(scene.Editor);
|
||||
var answer = await Uploads.StartAsync(scene.Editor, scene.Photo, sequenceId: scene.Target, expected: HttpStatusCode.OK);
|
||||
Assert.True(answer.AlreadyExists);
|
||||
Assert.False(answer.Copied);
|
||||
Assert.Equal(scene.Held.FrameId, answer.FrameId);
|
||||
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
||||
Assert.Equal(used, await UsedAsync(scene.Editor));
|
||||
}
|
||||
|
||||
/// <summary>The copy lands in the target with the photo's bytes, charged once, announced, and answered from the target after.</summary>
|
||||
internal static async Task CopiedAsync(TestServer server, TestContext context)
|
||||
{
|
||||
var scene = await SceneAsync(server);
|
||||
var used = await UsedAsync(scene.Editor);
|
||||
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
||||
|
||||
StartUploadResponse answer;
|
||||
using (var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target))
|
||||
{
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
||||
}
|
||||
Assert.True(answer.AlreadyExists && answer.Copied);
|
||||
Assert.Null(answer.UploadId);
|
||||
var frame = Assert.NotNull(await server.Database.Frames.GetAsync(Assert.NotNull(answer.FrameId), context.Cancellation));
|
||||
Assert.Equal(scene.Target, frame.SequenceId);
|
||||
Assert.Equal(Uploads.Sha(scene.Photo), frame.Sha256);
|
||||
await Http.BytesEqualAsync(scene.Photo, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, frame.RelativePath)));
|
||||
Assert.NotNull(await server.Database.Frames.GetAsync(scene.Held.FrameId, context.Cancellation));
|
||||
var request = Assert.NotNull(server.Services.GetRequiredService<FakeArchiveIngest>().Requests.LastOrDefault());
|
||||
Assert.Equal(scene.Target, request.SequenceId);
|
||||
Assert.True(request.RefuseRemoved, "the copy refuses a photo the target deleted");
|
||||
Assert.Equal(scene.Photo.Length, request.Size);
|
||||
Assert.True(request.StagedPath.StartsWith(server.Options.StagingPath, StringComparison.Ordinal), request.StagedPath);
|
||||
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
|
||||
Assert.Equal(used + scene.Photo.Length, await UsedAsync(scene.Editor));
|
||||
|
||||
using (var wait = CancellationTokenSource.CreateLinkedTokenSource(context.Cancellation))
|
||||
{
|
||||
wait.CancelAfter(TimeSpan.FromSeconds(10));
|
||||
while (await events.Reader.ReadAsync(wait.Token) is var e)
|
||||
{
|
||||
if (e is SequenceChangedEvent { Change: "frames" } changed && changed.SequenceId == scene.Target) break;
|
||||
}
|
||||
}
|
||||
|
||||
// Asked again, the target answers with its own copy, charged nothing more.
|
||||
using (var again = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target))
|
||||
{
|
||||
var repeated = await again.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
||||
Assert.True(repeated.AlreadyExists);
|
||||
Assert.False(repeated.Copied);
|
||||
Assert.Equal(frame.Id, repeated.FrameId);
|
||||
}
|
||||
Assert.Equal(used + scene.Photo.Length, await UsedAsync(scene.Editor));
|
||||
Assert.Count(1, await FramesOfAsync(server, scene.Target, scene.Photo));
|
||||
}
|
||||
|
||||
/// <summary>An editor's sequence A holding a 5,000-byte photo, and the editor's sequence B (the target) holding another photo.</summary>
|
||||
private static async Task<Scene> SceneAsync(TestServer server, long quota = 0)
|
||||
{
|
||||
var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: quota);
|
||||
var photo = Uploads.Photo(5000, 901);
|
||||
var held = await Uploads.UploadAsync(editor, photo, "IMG_0001.CR3", sessionKey: "a", capturedAt: Shot);
|
||||
var target = (await Uploads.UploadAsync(editor, Uploads.Photo(4000, 902), "IMG_0100.CR3", sessionKey: "b")).SequenceId;
|
||||
return new Scene(editor, photo, held, target);
|
||||
}
|
||||
|
||||
private static Task<HttpResponseMessage> StartIntoAsync(HttpClient client, byte[] data, Guid? target, bool restore = false) =>
|
||||
client.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(target, "IMG_0001.CR3", data.Length, Uploads.Sha(data), Shot, null, restore, IntoTarget: true),
|
||||
ProtocolJson.Default.StartUploadRequest);
|
||||
|
||||
private static async Task<long> UsedAsync(HttpClient client)
|
||||
{
|
||||
using var me = await client.GetAsync("/api/v1/auth/me");
|
||||
return (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes;
|
||||
}
|
||||
|
||||
private static async Task<IReadOnlyList<Frame>> FramesOfAsync(TestServer server, Guid sequenceId, byte[] photo) =>
|
||||
[.. (await server.Database.Frames.FindBySha256Async(Uploads.Sha(photo))).Where(f => f.SequenceId == sequenceId)];
|
||||
|
||||
private sealed record Scene(HttpClient Editor, byte[] Photo, CompleteUploadResponse Held, Guid Target);
|
||||
|
||||
/// <summary>The fake ingest, with a step that runs once just before the next commit (another writer filing the same photo).</summary>
|
||||
private sealed class RacingIngest(FakeArchiveIngest inner) : IArchiveIngest
|
||||
{
|
||||
public FakeArchiveIngest Inner => inner;
|
||||
|
||||
public Func<IngestRequest, Task>? Before { get; set; }
|
||||
|
||||
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (Before is { } before)
|
||||
{
|
||||
Before = null;
|
||||
await before(request);
|
||||
}
|
||||
return await inner.CommitAsync(request, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user