using System.Collections.Concurrent; using System.IO; using System.Net; using System.Net.Http; using System.Net.Http.Json; using System.Security.Cryptography; using Chronos.App.Import; using Chronos.Core.Import; using Chronos.Protocol; using Chronos.Protocol.Client; using Chronos.Testing; using static Chronos.App.Tests.Import.ImportTestData; namespace Chronos.App.Tests.Import; /// The "Server" destination over the real protocol client against an in-memory upload endpoint. [Category("import")] public sealed class ServerImportSinkTests(TestContext context) { [Test] public async Task EveryPhotoIsUploadedInChunksUnderItsCardNameAndVerifiedByHash() { var card = Card(context.CreateTempDirectory("card"), 8, Start, Start.AddHours(2)); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 256); using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); plan = plan with { Sessions = [plan.Sessions[0] with { Title = "Harbour sunset" }, .. plan.Sessions.Skip(1)] }; var outcome = await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation); Assert.Equal(16, outcome.Copied); Assert.True(outcome.IsComplete); Assert.Equal(16, server.Committed.Count); foreach (var item in plan.Items) { var bytes = File.ReadAllBytes(Path.Combine(card, item.SourcePath)); var upload = server.Committed.Single(u => u.FileName == Path.GetFileName(item.SourcePath)); Assert.BytesEqual(bytes, upload.Bytes); Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(bytes)), upload.Sha256); Assert.True(upload.Chunks > 1, "files larger than a chunk travel in several chunks"); } Assert.SequenceEqual(plan.Sessions.Select(s => s.SessionKey).Order(StringComparer.Ordinal), server.Committed.Select(u => u.SessionKey).Distinct().Order(StringComparer.Ordinal)); Assert.Equal("Harbour sunset", server.Titles.Single().Value); Assert.Empty(Directory.EnumerateFileSystemEntries(staging)); } [Test] public async Task AWholeFolderImportedAgainKeepsTheNameItsSequenceWasGivenOnTheServer() { var folder = Card(Path.Combine(context.CreateTempDirectory("folder"), "Etna"), 4, Start); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 4096); using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var destination = new UnindexedDestination(ImportDestinationKind.Server, staging); var options = new ImportChoices { Grouping = ImportGrouping.WholeFolder }.ToOptions(); var first = await destination.PlanAsync(folder, options, null, null, context.Cancellation); await LocalImport.RunAsync(first, new ServerImportSink(client, staging), "Etna", verify: true, null, context.Cancellation); var (sequence, title) = server.Titles.Single(); Assert.Equal("Etna", title, "a new sequence takes the folder's name"); // Renamed on the server; a photo added to the folder joins the same sequence and leaves its name alone. server.Titles[sequence] = "Etna 2026 eruption"; var photos = Path.Combine(folder, "DCIM", "100CANON"); File.WriteAllBytes(Path.Combine(photos, "IMG_0005.JPG"), Jpeg(Start.AddMinutes(10), 5)); var again = await destination.PlanAsync(folder, options, null, null, context.Cancellation); var outcome = await LocalImport.RunAsync(again, new ServerImportSink(client, staging), "Etna", verify: true, null, context.Cancellation); Assert.Equal((1, 4), (outcome.Copied, outcome.Duplicates)); Assert.Equal("Etna 2026 eruption", server.Titles[sequence]); Assert.Equal(1, server.TitleUpdates, "only the new sequence was named"); // A title the user types still names it. File.WriteAllBytes(Path.Combine(photos, "IMG_0006.JPG"), Jpeg(Start.AddMinutes(20), 6)); var typed = ImportPlans.WithTitles(await destination.PlanAsync(folder, options, null, null, context.Cancellation), new Dictionary { [again.Sessions[0].SessionKey] = "Etna at night" }); await LocalImport.RunAsync(typed, new ServerImportSink(client, staging), "Etna", verify: true, null, context.Cancellation); Assert.Equal("Etna at night", server.Titles.Single().Value); } [Test] public async Task PhotosTheServerAlreadyHoldsAreNotSentAgain() { var card = Card(context.CreateTempDirectory("card"), 8, Start); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 4096); using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var destination = new UnindexedDestination(ImportDestinationKind.Server, staging); var plan = await destination.PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation); var chunks = server.ChunkCount; var again = await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation); Assert.Equal(0, again.Copied); Assert.Equal(8, again.Duplicates); Assert.Equal(chunks, server.ChunkCount); } [Test] public async Task AFileTheServerRefusesIsListedAndTheImportGoesOn() { var card = Card(context.CreateTempDirectory("card"), 8, Start); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 4096) { CorruptFile = "IMG_0003.JPG" }; using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); var sink = new ServerImportSink(client, staging); var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report); Assert.Equal(7, outcome.Copied); Assert.Equal(7, server.Committed.Count); Assert.Equal(new ImportFailure("DCIM/100CANON/IMG_0003.JPG", "Hash mismatch"), outcome.Failures.Single()); Assert.Empty(Directory.EnumerateFileSystemEntries(staging)); } [Test] public async Task AFileAProxyRefusesIsListedWithTheCatalogsSentence() { var card = Card(context.CreateTempDirectory("card"), 8, Start); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 4096) { ProxyRefusesFile = "IMG_0005.JPG" }; using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); var sink = new ServerImportSink(client, staging); var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report); Assert.Equal(7, outcome.Copied); // The proxy's page says "Request Entity Too Large" in English: the photo reads the catalog's sentence instead. Assert.Equal(new ImportFailure("DCIM/100CANON/IMG_0005.JPG", Chronos.Core.Localization.Loc.L("Server.Error.TooLarge")), outcome.Failures.Single()); } [Test] public async Task ALostSignInStopsTheImportAndKeepsWhatWasUploaded() { var card = Card(context.CreateTempDirectory("card"), 8, Start); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 4096) { SignedOutAfter = 3 }; using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); var sink = new ServerImportSink(client, staging); var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report); Assert.Equal(3, outcome.Copied, "the uploads the server committed before the sign-in was lost"); Assert.Equal(5, outcome.Failures.Count); Assert.Equal("Sign in again", outcome.Failures[0].Message, "the server's sentence"); Assert.False(outcome.IsComplete); } [Test] public async Task TheQuotaIsTheRoomOfAServerImport() { var card = Card(context.CreateTempDirectory("card"), 8, Start); var staging = context.CreateTempDirectory("staging"); var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging) { ServerSpace = 1_000 } .PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); Assert.Equal(1_000L, plan.AvailableBytes); Assert.False(ImportPlans.Totals(plan, verify: true).HasEnoughSpace, "eight photos do not fit in 1 KB of quota"); var unlimited = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); Assert.Null(unlimited.AvailableBytes); Assert.True(ImportPlans.Totals(unlimited, verify: true).HasEnoughSpace); } [Test] public async Task AnImportIntoTheServerIsReportedAsCardEventsFromStartToFinish() { var card = Card(context.CreateTempDirectory("card"), 8, Start); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 4096); using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); var progress = new List(); var outcome = await new ServerImportSink(client, staging).RunAsync(plan, "EOS R5 · card A", true, new SyncProgress(progress.Add), context.Cancellation); Assert.True(outcome.IsComplete); Assert.NotEmpty(progress); var reports = server.Cards.ToList(); Assert.Equal(CardImportStates.Started, reports[0].State); Assert.Equal((CardImportStates.Finished, plan.FileCount, plan.FileCount, 0), (reports[^1].State, reports[^1].Done, reports[^1].Total, reports[^1].Failed)); Assert.True(reports.All(r => r.ImportId == plan.Id && r.Card == "EOS R5 · card A"), "every report names the import"); Assert.True(reports.Skip(1).SkipLast(1).All(r => r.State == CardImportStates.Progress), "progress in between"); } [Test] public async Task AServerWithoutCardEventsStillTakesTheImport() { var card = Card(context.CreateTempDirectory("card"), 8, Start); var staging = context.CreateTempDirectory("staging"); var server = new FakeUploadServer(chunkSize: 4096) { CardsUnsupported = true }; using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation); var outcome = await new ServerImportSink(client, staging).RunAsync(plan, "EOS R5 · card A", true, null, context.Cancellation); Assert.Equal(8, outcome.Copied); Assert.Equal(1, server.CardAttempts, "a server that answers 404 gets no further reports"); } [Test] public async Task ReportsWaitingForASlowServerAreReplacedByNewerOnes() { var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var server = new FakeUploadServer(chunkSize: 4096) { CardGate = gate.Task }; using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var time = new ManualTime(); var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100, time); reporter.Started(); await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10)); reporter.Report(Progress(10)); reporter.Report(Progress(20)); time.Advance(TimeSpan.FromSeconds(1)); reporter.Report(Progress(30)); var finished = reporter.FinishedAsync(null); gate.SetResult(); await finished.WaitAsync(TimeSpan.FromSeconds(10)); // The start was on its way; progress 10 waited and was replaced by 30 (20 came within the second), then by the end. Assert.SequenceEqual([(CardImportStates.Started, 0), (CardImportStates.Finished, 30)], server.Cards.Select(r => (r.State, r.Done))); // Nothing follows the end, not even progress a second later. time.Advance(TimeSpan.FromSeconds(1)); reporter.Report(Progress(40)); await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(10)); Assert.Equal(2, server.CardAttempts); } [Test] public async Task AnImportOverDoesNotWaitForAServerThatDoesNotAnswer() { var server = new FakeUploadServer(chunkSize: 4096) { CardGate = new TaskCompletionSource().Task }; using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" }; var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100); reporter.Started(); await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10)); var clock = System.Diagnostics.Stopwatch.StartNew(); await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(8)); // The start hangs (each report would wait 10 s): the end gives both up after two seconds. Assert.True(clock.Elapsed < TimeSpan.FromSeconds(6), $"finished after {clock.Elapsed}"); Assert.Empty(server.Cards); for (var i = 0; i < 25 && server.CardAttempts < 2; i++) await Task.Delay(20); Assert.Equal(1, server.CardAttempts, "the end was never sent once given up"); } private static ImportProgress Progress(int done) => new(done, 100, done * 10L, 1000, done, 0, 0, null); /// Reports synchronously (Progress<T> posts and may arrive late). private sealed class SyncProgress(Action report) : IProgress { public void Report(T value) => report(value); } private sealed class ManualTime : TimeProvider { private long _ticks; public override long TimestampFrequency => TimeSpan.TicksPerSecond; public override long GetTimestamp() => Interlocked.Read(ref _ticks); public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks); } /// Start, chunk and complete of BRIEF §14.4, plus the sequence's title and card reports; the hash is checked at completion. private sealed class FakeUploadServer(int chunkSize) : HttpMessageHandler { private readonly Dictionary _open = []; private readonly Dictionary _sequences = new(StringComparer.Ordinal); private readonly Dictionary _frames = new(StringComparer.Ordinal); public List Committed { get; } = []; /// Each sequence's title on the server (a test may rename one, as a user would there). public Dictionary Titles { get; } = []; public int TitleUpdates { get; private set; } public int ChunkCount { get; private set; } /// Flip a byte of every chunk of this file (a damaged transfer). public string? CorruptFile { get; init; } /// A reverse proxy in front of the server refuses the chunks of this file: 413 with its own HTML page. public string? ProxyRefusesFile { get; init; } /// Answer 401 to new uploads once this many were committed (the sign-in expired). public int? SignedOutAfter { get; init; } /// Card reports taken, in order (they arrive on another thread than the uploads). public ConcurrentQueue Cards { get; } = new(); public int CardAttempts => _cardAttempts; /// A server from before card events: 404. public bool CardsUnsupported { get; init; } /// Card reports wait for this (a slow server); says when the first one is waiting. public Task? CardGate { get; init; } public TaskCompletionSource CardArrived { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); private int _cardAttempts; protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { var path = request.RequestUri!.AbsolutePath.Trim('/').Split('/'); if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads", "cards"]) { Interlocked.Increment(ref _cardAttempts); if (CardsUnsupported) return new HttpResponseMessage(HttpStatusCode.NotFound); var report = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.CardImportReport, cancellationToken))!; CardArrived.TrySetResult(); if (CardGate is { } gate) await gate.WaitAsync(cancellationToken); Cards.Enqueue(report); return new HttpResponseMessage(HttpStatusCode.NoContent); } if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads"]) { var start = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.StartUploadRequest, cancellationToken))!; if (Committed.Count >= SignedOutAfter) return Json(new ApiError("unauthorized", "Sign in again"), ProtocolJson.Default.ApiError, HttpStatusCode.Unauthorized); if (_frames.TryGetValue(start.Sha256, out var existing)) return Json(new StartUploadResponse(null, chunkSize, [], true, existing), ProtocolJson.Default.StartUploadResponse); var upload = new Upload(Guid.NewGuid(), start.FileName, start.Size, start.Sha256, start.SessionKey ?? "user"); _open[upload.Id] = upload; return Json(new StartUploadResponse(upload.Id, chunkSize, [], false, null), ProtocolJson.Default.StartUploadResponse); } if (request.Method == HttpMethod.Put && path is ["api", "v1", "uploads", var id, "chunks", var n]) { var upload = _open[Guid.Parse(id)]; if (upload.FileName == ProxyRefusesFile) { return new HttpResponseMessage(HttpStatusCode.RequestEntityTooLarge) { Content = new StringContent("

413 Request Entity Too Large

", System.Text.Encoding.UTF8, "text/html"), }; } var bytes = await request.Content!.ReadAsByteArrayAsync(cancellationToken); if (upload.FileName == CorruptFile) bytes[0] ^= 0xFF; upload.Parts[int.Parse(n, System.Globalization.CultureInfo.InvariantCulture)] = bytes; ChunkCount++; return Json(new ChunkReceived(int.Parse(n, System.Globalization.CultureInfo.InvariantCulture), [.. upload.Parts.Keys]), ProtocolJson.Default.ChunkReceived); } if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads", var done, "complete"]) { var upload = _open[Guid.Parse(done)]; if (Convert.ToHexStringLower(SHA256.HashData(upload.Bytes)) != upload.Sha256) return Json(new ApiError("hash_mismatch", "Hash mismatch"), ProtocolJson.Default.ApiError, HttpStatusCode.UnprocessableEntity); _open.Remove(upload.Id); Committed.Add(upload); var frame = Guid.NewGuid(); _frames[upload.Sha256] = frame; if (!_sequences.TryGetValue(upload.SessionKey, out var sequence)) _sequences[upload.SessionKey] = sequence = Guid.NewGuid(); return Json(new CompleteUploadResponse(frame, sequence, Committed.Count(u => u.SessionKey == upload.SessionKey) - 1), ProtocolJson.Default.CompleteUploadResponse); } if (request.Method == HttpMethod.Get && path is ["api", "v1", "sequences", var read]) return Json(Sequence(Guid.Parse(read), Titles.GetValueOrDefault(Guid.Parse(read))), ProtocolJson.Default.SequenceDto); if (request.Method == HttpMethod.Patch && path is ["api", "v1", "sequences", var sequenceId]) { var update = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.UpdateSequenceRequest, cancellationToken))!; Titles[Guid.Parse(sequenceId)] = update.Title!; TitleUpdates++; return Json(Sequence(Guid.Parse(sequenceId), update.Title), ProtocolJson.Default.SequenceDto); } return new HttpResponseMessage(HttpStatusCode.NotFound); } private static SequenceDto Sequence(Guid id, string? title) => new(id, title, title ?? "f", "f", null, null, null, null, null, 0, null, 0, 0, 0, null, [], 0, null, 1, 1, 0, null, DateTimeOffset.UtcNow); private static HttpResponseMessage Json(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo type, HttpStatusCode status = HttpStatusCode.OK) => new(status) { Content = JsonContent.Create(value, type) }; public sealed record Upload(Guid Id, string FileName, long Size, string Sha256, string SessionKey) { public SortedDictionary Parts { get; } = []; public byte[] Bytes => [.. Parts.Values.SelectMany(p => p)]; public int Chunks => Parts.Count; } } }