CI / Build and test (push) Successful in 13m44s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 16s
Release / Build, test and publish (push) Successful in 26m34s
CI / PostgreSQL 16 integration (push) Failing after 17m36s
CI / PostgreSQL 17 integration (push) Failing after 15m36s
CI / Docker image (push) Successful in 1m12s
Release / Server image (push) Failing after 15s
The same time-lapse on a server and on this PC under different ids, or twice on this PC, is recognised, compared and settled as the owner asked in owner review 6. Matches are found after each sync refresh, through the server's new photo lookup, and after the jobs that change photos. The default is to ask every time. Server › Matches offers: - Keep apart, which is remembered; - Link only and Unlink; - the Compare dialog: photos, details line by line, projects and the plan. Integrate completes both sides. Photos default to Union, projects are copied across, and a losing version goes to the trash only after a hash check. Afterwards new photos move by themselves, and deletions wait for the user. A merge of two local sequences verifies every photo before the other sequence goes to its archive's trash. Upload all local skips suspected matches and flags them. Migration 0012 adds the found_copies and copy_probes tables. Jobs 25 (Integrate) and 26 (MergeSequences) are new, and so are the server capabilities photo-lookup and upload-into. Refs #73 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
299 lines
15 KiB
C#
299 lines
15 KiB
C#
using System.Net;
|
|
using System.Net.Http.Json;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data;
|
|
using Chronos.Data.Archiving;
|
|
using Chronos.Protocol;
|
|
using Chronos.Server.Events;
|
|
using Chronos.Server.Tests.Api;
|
|
using Chronos.Testing;
|
|
|
|
namespace Chronos.Server.Tests.Storage;
|
|
|
|
/// <summary>
|
|
/// <see cref="StartUploadRequest.IntoTarget"/> (owner review 6, #73): a photo another sequence of the served archive holds is
|
|
/// copied into the target by the server itself, authorised and scoped on the server.
|
|
/// </summary>
|
|
[Category("server")]
|
|
public sealed class UploadIntoTests(TestContext context)
|
|
{
|
|
private static readonly DateTime Shot = new(2026, 9, 28, 18, 2, 11, 250, DateTimeKind.Unspecified);
|
|
|
|
[Test]
|
|
public async Task APhotoHeldElsewhereIsCopiedIntoTheTargetAndChargedOnce()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
await CopiedAsync(server, context);
|
|
}
|
|
|
|
[Test]
|
|
[RequiresEnvironment("CHRONOS_TEST_PG")]
|
|
public async Task APhotoHeldElsewhereIsCopiedIntoTheTargetAndChargedOnceOnPostgres()
|
|
{
|
|
await using var schema = await PgSchema.CreateAsync(context);
|
|
await using var server = await schema.StartAsync(context);
|
|
Assert.Equal(Dialect.Postgres, server.Database.Dialect);
|
|
await CopiedAsync(server, context);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ACommitThatFindsThePhotoFiledMeanwhileChargesNothing()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context, services: s =>
|
|
{
|
|
s.AddSingleton<RacingIngest>();
|
|
s.AddSingleton<IArchiveIngest>(sp => sp.GetRequiredService<RacingIngest>());
|
|
});
|
|
var scene = await SceneAsync(server);
|
|
var racing = server.Services.GetRequiredService<RacingIngest>();
|
|
Guid? raced = null;
|
|
// Another writer files the same photo into the target just before this commit: the ingest answers a duplicate.
|
|
racing.Before = async request =>
|
|
{
|
|
var copy = request.StagedPath + ".race";
|
|
File.Copy(request.StagedPath, copy);
|
|
raced = (await racing.Inner.CommitAsync(request with { StagedPath = copy }, context.Cancellation)).FrameId;
|
|
};
|
|
var used = await UsedAsync(scene.Editor);
|
|
|
|
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target);
|
|
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
|
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
|
Assert.True(answer.AlreadyExists && answer.Copied);
|
|
Assert.Equal(Assert.NotNull(raced), answer.FrameId);
|
|
Assert.Equal(used, await UsedAsync(scene.Editor));
|
|
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheCopyIsRefusedBeyondTheQuota()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
// The two uploads of the scene take 9,000 bytes; copying the 5,000-byte photo would take the account beyond 10,000.
|
|
var scene = await SceneAsync(server, quota: 10_000);
|
|
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target);
|
|
await response.ExpectErrorAsync(HttpStatusCode.Forbidden, "quota_exceeded");
|
|
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
|
}
|
|
|
|
[Test]
|
|
public async Task OnlyTheOwnerAnAdminOrAnEditorMayUploadIntoATarget()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
var (uploader, _, _) = await server.UserAsync("roof-cam", "uploader");
|
|
|
|
// An upload-only account may not file into another account's sequence.
|
|
using (var refused = await StartIntoAsync(uploader, scene.Photo, scene.Target))
|
|
await refused.ExpectErrorAsync(HttpStatusCode.Forbidden, "forbidden");
|
|
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
|
|
|
// Into its own sequence it may: the server copies the photo the editor's sequence holds.
|
|
var own = (await Uploads.UploadAsync(uploader, Uploads.Photo(3000, 903), "IMG_0200.CR3", sessionKey: "roof")).SequenceId;
|
|
using (var allowed = await StartIntoAsync(uploader, scene.Photo, own))
|
|
Assert.True((await allowed.ReadAsync(ProtocolJson.Default.StartUploadResponse)).Copied);
|
|
Assert.Count(1, await FramesOfAsync(server, own, scene.Photo));
|
|
|
|
// An administrator may.
|
|
var admin = await server.AdminAsync();
|
|
using (var byAdmin = await StartIntoAsync(admin, scene.Photo, scene.Target))
|
|
Assert.True((await byAdmin.ReadAsync(ProtocolJson.Default.StartUploadResponse)).Copied);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ATargetOfAnotherArchiveIsUnknown()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
var foreign = await PhotoLookupTests.ForeignFrameAsync(server, context, Uploads.Photo(2000, 904));
|
|
using (var other = await StartIntoAsync(scene.Editor, scene.Photo, foreign.SequenceId))
|
|
Assert.Equal("sequence", (await other.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found")).Details);
|
|
using (var unknown = await StartIntoAsync(scene.Editor, scene.Photo, Guid.CreateVersion7()))
|
|
await unknown.ExpectErrorAsync(HttpStatusCode.NotFound, "not_found");
|
|
Assert.Empty(await FramesOfAsync(server, foreign.SequenceId, scene.Photo));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnUploadIntoATargetNamesItAndNeverRestores()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
using (var restore = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target, restore: true))
|
|
Assert.Equal("restore", (await restore.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
|
using (var none = await StartIntoAsync(scene.Editor, scene.Photo, null))
|
|
Assert.Equal("sequenceId", (await none.ExpectErrorAsync(HttpStatusCode.BadRequest, "bad_request")).Details);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ATargetThatHoldsThePhotoAnswersItsFrame()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Held.SequenceId);
|
|
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
|
Assert.True(answer.AlreadyExists);
|
|
Assert.False(answer.Copied);
|
|
Assert.Equal(scene.Held.FrameId, answer.FrameId);
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoTheTargetDeletedStaysDeleted()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
await server.Database.FrameRemovals.SyncSequenceAsync(scene.Target,
|
|
[new FrameRemoval(scene.Target, Uploads.Sha(scene.Photo), null, RemovalReason.Deleted, DateTimeOffset.UtcNow)], context.Cancellation);
|
|
var requests = server.Services.GetRequiredService<FakeArchiveIngest>().Requests.Count;
|
|
|
|
using var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target);
|
|
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
|
Assert.True(answer.AlreadyExists);
|
|
Assert.Null(answer.FrameId);
|
|
Assert.False(answer.Copied);
|
|
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
|
Assert.Equal(requests, server.Services.GetRequiredService<FakeArchiveIngest>().Requests.Count);
|
|
}
|
|
|
|
[Test]
|
|
public async Task APhotoOnlyAnotherArchiveHoldsIsUploaded()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
var elsewhere = Uploads.Photo(2500, 905);
|
|
await PhotoLookupTests.ForeignFrameAsync(server, context, elsewhere);
|
|
|
|
using var response = await StartIntoAsync(scene.Editor, elsewhere, scene.Target);
|
|
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
|
|
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
|
Assert.NotNull(answer.UploadId);
|
|
Assert.False(answer.AlreadyExists || answer.Copied);
|
|
}
|
|
|
|
[Test]
|
|
public async Task ADamagedSourceLeavesTheClientToSendThePhoto()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
// The file keeps its size, so the frame reads as intact; its bytes are no longer the photo.
|
|
var source = Assert.NotNull(await server.Database.Frames.GetAsync(scene.Held.FrameId));
|
|
await File.WriteAllBytesAsync(Path.Combine(server.Options.ArchivePath, source.RelativePath), Uploads.Photo(scene.Photo.Length, 906), context.Cancellation);
|
|
|
|
using (var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target))
|
|
{
|
|
Assert.Equal(HttpStatusCode.Created, response.StatusCode);
|
|
var answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
|
Assert.NotNull(answer.UploadId);
|
|
Assert.False(answer.Copied);
|
|
}
|
|
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
|
|
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
|
}
|
|
|
|
[Test]
|
|
public async Task WithoutIntoTargetACopyElsewhereStillStandsIn()
|
|
{
|
|
await using var server = await TestServer.StartAsync(context);
|
|
var scene = await SceneAsync(server);
|
|
var used = await UsedAsync(scene.Editor);
|
|
var answer = await Uploads.StartAsync(scene.Editor, scene.Photo, sequenceId: scene.Target, expected: HttpStatusCode.OK);
|
|
Assert.True(answer.AlreadyExists);
|
|
Assert.False(answer.Copied);
|
|
Assert.Equal(scene.Held.FrameId, answer.FrameId);
|
|
Assert.Empty(await FramesOfAsync(server, scene.Target, scene.Photo));
|
|
Assert.Equal(used, await UsedAsync(scene.Editor));
|
|
}
|
|
|
|
/// <summary>The copy lands in the target with the photo's bytes, charged once, announced, and answered from the target after.</summary>
|
|
internal static async Task CopiedAsync(TestServer server, TestContext context)
|
|
{
|
|
var scene = await SceneAsync(server);
|
|
var used = await UsedAsync(scene.Editor);
|
|
using var events = server.Services.GetRequiredService<EventHub>().Subscribe();
|
|
|
|
StartUploadResponse answer;
|
|
using (var response = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target))
|
|
{
|
|
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
|
answer = await response.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
|
}
|
|
Assert.True(answer.AlreadyExists && answer.Copied);
|
|
Assert.Null(answer.UploadId);
|
|
var frame = Assert.NotNull(await server.Database.Frames.GetAsync(Assert.NotNull(answer.FrameId), context.Cancellation));
|
|
Assert.Equal(scene.Target, frame.SequenceId);
|
|
Assert.Equal(Uploads.Sha(scene.Photo), frame.Sha256);
|
|
await Http.BytesEqualAsync(scene.Photo, File.ReadAllBytesAsync(Path.Combine(server.Options.ArchivePath, frame.RelativePath)));
|
|
Assert.NotNull(await server.Database.Frames.GetAsync(scene.Held.FrameId, context.Cancellation));
|
|
var request = Assert.NotNull(server.Services.GetRequiredService<FakeArchiveIngest>().Requests.LastOrDefault());
|
|
Assert.Equal(scene.Target, request.SequenceId);
|
|
Assert.True(request.RefuseRemoved, "the copy refuses a photo the target deleted");
|
|
Assert.Equal(scene.Photo.Length, request.Size);
|
|
Assert.True(request.StagedPath.StartsWith(server.Options.StagingPath, StringComparison.Ordinal), request.StagedPath);
|
|
Assert.Empty(Directory.EnumerateFiles(server.Options.StagingPath));
|
|
Assert.Equal(used + scene.Photo.Length, await UsedAsync(scene.Editor));
|
|
|
|
using (var wait = CancellationTokenSource.CreateLinkedTokenSource(context.Cancellation))
|
|
{
|
|
wait.CancelAfter(TimeSpan.FromSeconds(10));
|
|
while (await events.Reader.ReadAsync(wait.Token) is var e)
|
|
{
|
|
if (e is SequenceChangedEvent { Change: "frames" } changed && changed.SequenceId == scene.Target) break;
|
|
}
|
|
}
|
|
|
|
// Asked again, the target answers with its own copy, charged nothing more.
|
|
using (var again = await StartIntoAsync(scene.Editor, scene.Photo, scene.Target))
|
|
{
|
|
var repeated = await again.ReadAsync(ProtocolJson.Default.StartUploadResponse);
|
|
Assert.True(repeated.AlreadyExists);
|
|
Assert.False(repeated.Copied);
|
|
Assert.Equal(frame.Id, repeated.FrameId);
|
|
}
|
|
Assert.Equal(used + scene.Photo.Length, await UsedAsync(scene.Editor));
|
|
Assert.Count(1, await FramesOfAsync(server, scene.Target, scene.Photo));
|
|
}
|
|
|
|
/// <summary>An editor's sequence A holding a 5,000-byte photo, and the editor's sequence B (the target) holding another photo.</summary>
|
|
private static async Task<Scene> SceneAsync(TestServer server, long quota = 0)
|
|
{
|
|
var (editor, _, _) = await server.UserAsync("alberto", "editor", quotaBytes: quota);
|
|
var photo = Uploads.Photo(5000, 901);
|
|
var held = await Uploads.UploadAsync(editor, photo, "IMG_0001.CR3", sessionKey: "a", capturedAt: Shot);
|
|
var target = (await Uploads.UploadAsync(editor, Uploads.Photo(4000, 902), "IMG_0100.CR3", sessionKey: "b")).SequenceId;
|
|
return new Scene(editor, photo, held, target);
|
|
}
|
|
|
|
private static Task<HttpResponseMessage> StartIntoAsync(HttpClient client, byte[] data, Guid? target, bool restore = false) =>
|
|
client.PostAsJsonAsync("/api/v1/uploads", new StartUploadRequest(target, "IMG_0001.CR3", data.Length, Uploads.Sha(data), Shot, null, restore, IntoTarget: true),
|
|
ProtocolJson.Default.StartUploadRequest);
|
|
|
|
private static async Task<long> UsedAsync(HttpClient client)
|
|
{
|
|
using var me = await client.GetAsync("/api/v1/auth/me");
|
|
return (await me.ReadAsync(ProtocolJson.Default.UserDto)).UsedBytes;
|
|
}
|
|
|
|
private static async Task<IReadOnlyList<Frame>> FramesOfAsync(TestServer server, Guid sequenceId, byte[] photo) =>
|
|
[.. (await server.Database.Frames.FindBySha256Async(Uploads.Sha(photo))).Where(f => f.SequenceId == sequenceId)];
|
|
|
|
private sealed record Scene(HttpClient Editor, byte[] Photo, CompleteUploadResponse Held, Guid Target);
|
|
|
|
/// <summary>The fake ingest, with a step that runs once just before the next commit (another writer filing the same photo).</summary>
|
|
private sealed class RacingIngest(FakeArchiveIngest inner) : IArchiveIngest
|
|
{
|
|
public FakeArchiveIngest Inner => inner;
|
|
|
|
public Func<IngestRequest, Task>? Before { get; set; }
|
|
|
|
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
|
|
{
|
|
if (Before is { } before)
|
|
{
|
|
Before = null;
|
|
await before(request);
|
|
}
|
|
return await inner.CommitAsync(request, cancellationToken);
|
|
}
|
|
}
|
|
}
|