Files
Chronos/tests/Titano.App.Tests/Import/ServerImportSinkTests.cs
T
Claude 6e213d4997 fix(import): say what a proxy's reply means instead of its English reason phrase
A reply that is not the server's JSON (a reverse proxy's 502 page while the
container restarts, its 413 for a large body, a bare 500) reached the Import
screen as the HTTP reason phrase, in English, beside every photo. ErrorText now
uses an ApiError's message only when the server wrote it (a code that is not
TitanoClient's http_<status> fallback) and otherwise says what the status means:
too large, rate limited, not answering (gateway and timeout statuses) or a
server error. A card file that cannot be read while planning is listed with the
executor's catalog sentence instead of the system's text and full path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
2026-10-12 16:21:10 +00:00

345 lines
20 KiB
C#

using System.Collections.Concurrent;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Net.Http.Json;
using System.Security.Cryptography;
using Titano.App.Import;
using Titano.Core.Import;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Testing;
using static Titano.App.Tests.Import.ImportTestData;
namespace Titano.App.Tests.Import;
/// <summary>The "Server" destination over the real protocol client against an in-memory upload endpoint.</summary>
[Category("import")]
public sealed class ServerImportSinkTests(TestContext context)
{
[Test]
public async Task EveryPhotoIsUploadedInChunksUnderItsCardNameAndVerifiedByHash()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start, Start.AddHours(2));
var staging = context.CreateTempDirectory("staging");
var server = new FakeUploadServer(chunkSize: 256);
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
plan = plan with { Sessions = [plan.Sessions[0] with { Title = "Harbour sunset" }, .. plan.Sessions.Skip(1)] };
var outcome = await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation);
Assert.Equal(16, outcome.Copied);
Assert.True(outcome.IsComplete);
Assert.Equal(16, server.Committed.Count);
foreach (var item in plan.Items)
{
var bytes = File.ReadAllBytes(Path.Combine(card, item.SourcePath));
var upload = server.Committed.Single(u => u.FileName == Path.GetFileName(item.SourcePath));
Assert.BytesEqual(bytes, upload.Bytes);
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(bytes)), upload.Sha256);
Assert.True(upload.Chunks > 1, "files larger than a chunk travel in several chunks");
}
Assert.SequenceEqual(plan.Sessions.Select(s => s.SessionKey).Order(StringComparer.Ordinal),
server.Committed.Select(u => u.SessionKey).Distinct().Order(StringComparer.Ordinal));
Assert.Equal("Harbour sunset", server.Titles.Single().Value);
Assert.Empty(Directory.EnumerateFileSystemEntries(staging));
}
[Test]
public async Task PhotosTheServerAlreadyHoldsAreNotSentAgain()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start);
var staging = context.CreateTempDirectory("staging");
var server = new FakeUploadServer(chunkSize: 4096);
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var destination = new UnindexedDestination(ImportDestinationKind.Server, staging);
var plan = await destination.PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation);
var chunks = server.ChunkCount;
var again = await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation);
Assert.Equal(0, again.Copied);
Assert.Equal(8, again.Duplicates);
Assert.Equal(chunks, server.ChunkCount);
}
[Test]
public async Task AFileTheServerRefusesIsListedAndTheImportGoesOn()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start);
var staging = context.CreateTempDirectory("staging");
var server = new FakeUploadServer(chunkSize: 4096) { CorruptFile = "IMG_0003.JPG" };
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
var sink = new ServerImportSink(client, staging);
var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report);
Assert.Equal(7, outcome.Copied);
Assert.Equal(7, server.Committed.Count);
Assert.Equal(new ImportFailure("DCIM/100CANON/IMG_0003.JPG", "Hash mismatch"), outcome.Failures.Single());
Assert.Empty(Directory.EnumerateFileSystemEntries(staging));
}
[Test]
public async Task AFileAProxyRefusesIsListedWithTheCatalogsSentence()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start);
var staging = context.CreateTempDirectory("staging");
var server = new FakeUploadServer(chunkSize: 4096) { ProxyRefusesFile = "IMG_0005.JPG" };
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
var sink = new ServerImportSink(client, staging);
var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report);
Assert.Equal(7, outcome.Copied);
// The proxy's page says "Request Entity Too Large" in English: the photo reads the catalog's sentence instead.
Assert.Equal(new ImportFailure("DCIM/100CANON/IMG_0005.JPG", Titano.Core.Localization.Loc.L("Server.Error.TooLarge")), outcome.Failures.Single());
}
[Test]
public async Task ALostSignInStopsTheImportAndKeepsWhatWasUploaded()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start);
var staging = context.CreateTempDirectory("staging");
var server = new FakeUploadServer(chunkSize: 4096) { SignedOutAfter = 3 };
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
var sink = new ServerImportSink(client, staging);
var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report);
Assert.Equal(3, outcome.Copied, "the uploads the server committed before the sign-in was lost");
Assert.Equal(5, outcome.Failures.Count);
Assert.Equal("Sign in again", outcome.Failures[0].Message, "the server's sentence");
Assert.False(outcome.IsComplete);
}
[Test]
public async Task TheQuotaIsTheRoomOfAServerImport()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start);
var staging = context.CreateTempDirectory("staging");
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging) { ServerSpace = 1_000 }
.PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
Assert.Equal(1_000L, plan.AvailableBytes);
Assert.False(ImportPlans.Totals(plan, verify: true).HasEnoughSpace, "eight photos do not fit in 1 KB of quota");
var unlimited = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null,
context.Cancellation);
Assert.Null(unlimited.AvailableBytes);
Assert.True(ImportPlans.Totals(unlimited, verify: true).HasEnoughSpace);
}
[Test]
public async Task AnImportIntoTheServerIsReportedAsCardEventsFromStartToFinish()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start);
var staging = context.CreateTempDirectory("staging");
var server = new FakeUploadServer(chunkSize: 4096);
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
var progress = new List<ImportProgress>();
var outcome = await new ServerImportSink(client, staging).RunAsync(plan, "EOS R5 · card A", true, new SyncProgress<ImportProgress>(progress.Add),
context.Cancellation);
Assert.True(outcome.IsComplete);
Assert.NotEmpty(progress);
var reports = server.Cards.ToList();
Assert.Equal(CardImportStates.Started, reports[0].State);
Assert.Equal((CardImportStates.Finished, plan.FileCount, plan.FileCount, 0), (reports[^1].State, reports[^1].Done, reports[^1].Total, reports[^1].Failed));
Assert.True(reports.All(r => r.ImportId == plan.Id && r.Card == "EOS R5 · card A"), "every report names the import");
Assert.True(reports.Skip(1).SkipLast(1).All(r => r.State == CardImportStates.Progress), "progress in between");
}
[Test]
public async Task AServerWithoutCardEventsStillTakesTheImport()
{
var card = Card(context.CreateTempDirectory("card"), 8, Start);
var staging = context.CreateTempDirectory("staging");
var server = new FakeUploadServer(chunkSize: 4096) { CardsUnsupported = true };
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
var outcome = await new ServerImportSink(client, staging).RunAsync(plan, "EOS R5 · card A", true, null, context.Cancellation);
Assert.Equal(8, outcome.Copied);
Assert.Equal(1, server.CardAttempts, "a server that answers 404 gets no further reports");
}
[Test]
public async Task ReportsWaitingForASlowServerAreReplacedByNewerOnes()
{
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var server = new FakeUploadServer(chunkSize: 4096) { CardGate = gate.Task };
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var time = new ManualTime();
var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100, time);
reporter.Started();
await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10));
reporter.Report(Progress(10));
reporter.Report(Progress(20));
time.Advance(TimeSpan.FromSeconds(1));
reporter.Report(Progress(30));
var finished = reporter.FinishedAsync(null);
gate.SetResult();
await finished.WaitAsync(TimeSpan.FromSeconds(10));
// The start was on its way; progress 10 waited and was replaced by 30 (20 came within the second), then by the end.
Assert.SequenceEqual([(CardImportStates.Started, 0), (CardImportStates.Finished, 30)], server.Cards.Select(r => (r.State, r.Done)));
// Nothing follows the end, not even progress a second later.
time.Advance(TimeSpan.FromSeconds(1));
reporter.Report(Progress(40));
await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(10));
Assert.Equal(2, server.CardAttempts);
}
[Test]
public async Task AnImportOverDoesNotWaitForAServerThatDoesNotAnswer()
{
var server = new FakeUploadServer(chunkSize: 4096) { CardGate = new TaskCompletionSource().Task };
using var client = new TitanoClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100);
reporter.Started();
await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10));
var clock = System.Diagnostics.Stopwatch.StartNew();
await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(8));
// The start hangs (each report would wait 10 s): the end gives both up after two seconds.
Assert.True(clock.Elapsed < TimeSpan.FromSeconds(6), $"finished after {clock.Elapsed}");
Assert.Empty(server.Cards);
for (var i = 0; i < 25 && server.CardAttempts < 2; i++) await Task.Delay(20);
Assert.Equal(1, server.CardAttempts, "the end was never sent once given up");
}
private static ImportProgress Progress(int done) => new(done, 100, done * 10L, 1000, done, 0, 0, null);
/// <summary>Reports synchronously (Progress&lt;T&gt; posts and may arrive late).</summary>
private sealed class SyncProgress<T>(Action<T> report) : IProgress<T>
{
public void Report(T value) => report(value);
}
private sealed class ManualTime : TimeProvider
{
private long _ticks;
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
public override long GetTimestamp() => Interlocked.Read(ref _ticks);
public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks);
}
/// <summary>Start, chunk and complete of BRIEF §14.4, plus the title update and card reports; the hash is checked at completion.</summary>
private sealed class FakeUploadServer(int chunkSize) : HttpMessageHandler
{
private readonly Dictionary<Guid, Upload> _open = [];
private readonly Dictionary<string, Guid> _sequences = new(StringComparer.Ordinal);
private readonly Dictionary<string, Guid> _frames = new(StringComparer.Ordinal);
public List<Upload> Committed { get; } = [];
public Dictionary<Guid, string> Titles { get; } = [];
public int ChunkCount { get; private set; }
/// <summary>Flip a byte of every chunk of this file (a damaged transfer).</summary>
public string? CorruptFile { get; init; }
/// <summary>A reverse proxy in front of the server refuses the chunks of this file: 413 with its own HTML page.</summary>
public string? ProxyRefusesFile { get; init; }
/// <summary>Answer 401 to new uploads once this many were committed (the sign-in expired).</summary>
public int? SignedOutAfter { get; init; }
/// <summary>Card reports taken, in order (they arrive on another thread than the uploads).</summary>
public ConcurrentQueue<CardImportReport> Cards { get; } = new();
public int CardAttempts => _cardAttempts;
/// <summary>A server from before card events: 404.</summary>
public bool CardsUnsupported { get; init; }
/// <summary>Card reports wait for this (a slow server); <see cref="CardArrived"/> says when the first one is waiting.</summary>
public Task? CardGate { get; init; }
public TaskCompletionSource CardArrived { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _cardAttempts;
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var path = request.RequestUri!.AbsolutePath.Trim('/').Split('/');
if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads", "cards"])
{
Interlocked.Increment(ref _cardAttempts);
if (CardsUnsupported) return new HttpResponseMessage(HttpStatusCode.NotFound);
var report = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.CardImportReport, cancellationToken))!;
CardArrived.TrySetResult();
if (CardGate is { } gate) await gate.WaitAsync(cancellationToken);
Cards.Enqueue(report);
return new HttpResponseMessage(HttpStatusCode.NoContent);
}
if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads"])
{
var start = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.StartUploadRequest, cancellationToken))!;
if (Committed.Count >= SignedOutAfter)
return Json(new ApiError("unauthorized", "Sign in again"), ProtocolJson.Default.ApiError, HttpStatusCode.Unauthorized);
if (_frames.TryGetValue(start.Sha256, out var existing))
return Json(new StartUploadResponse(null, chunkSize, [], true, existing), ProtocolJson.Default.StartUploadResponse);
var upload = new Upload(Guid.NewGuid(), start.FileName, start.Size, start.Sha256, start.SessionKey ?? "user");
_open[upload.Id] = upload;
return Json(new StartUploadResponse(upload.Id, chunkSize, [], false, null), ProtocolJson.Default.StartUploadResponse);
}
if (request.Method == HttpMethod.Put && path is ["api", "v1", "uploads", var id, "chunks", var n])
{
var upload = _open[Guid.Parse(id)];
if (upload.FileName == ProxyRefusesFile)
{
return new HttpResponseMessage(HttpStatusCode.RequestEntityTooLarge)
{
Content = new StringContent("<html><body><h1>413 Request Entity Too Large</h1></body></html>", System.Text.Encoding.UTF8, "text/html"),
};
}
var bytes = await request.Content!.ReadAsByteArrayAsync(cancellationToken);
if (upload.FileName == CorruptFile) bytes[0] ^= 0xFF;
upload.Parts[int.Parse(n, System.Globalization.CultureInfo.InvariantCulture)] = bytes;
ChunkCount++;
return Json(new ChunkReceived(int.Parse(n, System.Globalization.CultureInfo.InvariantCulture), [.. upload.Parts.Keys]), ProtocolJson.Default.ChunkReceived);
}
if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads", var done, "complete"])
{
var upload = _open[Guid.Parse(done)];
if (Convert.ToHexStringLower(SHA256.HashData(upload.Bytes)) != upload.Sha256)
return Json(new ApiError("hash_mismatch", "Hash mismatch"), ProtocolJson.Default.ApiError, HttpStatusCode.UnprocessableEntity);
_open.Remove(upload.Id);
Committed.Add(upload);
var frame = Guid.NewGuid();
_frames[upload.Sha256] = frame;
if (!_sequences.TryGetValue(upload.SessionKey, out var sequence)) _sequences[upload.SessionKey] = sequence = Guid.NewGuid();
return Json(new CompleteUploadResponse(frame, sequence, Committed.Count(u => u.SessionKey == upload.SessionKey) - 1),
ProtocolJson.Default.CompleteUploadResponse);
}
if (request.Method == HttpMethod.Patch && path is ["api", "v1", "sequences", var sequenceId])
{
var update = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.UpdateSequenceRequest, cancellationToken))!;
Titles[Guid.Parse(sequenceId)] = update.Title!;
return Json(new SequenceDto(Guid.Parse(sequenceId), update.Title, update.Title!, "f", null, null, null, null, null, 0, null, 0, 0, 0, null, [], 0, null,
1, 1, 0, null, DateTimeOffset.UtcNow), ProtocolJson.Default.SequenceDto);
}
return new HttpResponseMessage(HttpStatusCode.NotFound);
}
private static HttpResponseMessage Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> type, HttpStatusCode status = HttpStatusCode.OK) =>
new(status) { Content = JsonContent.Create(value, type) };
public sealed record Upload(Guid Id, string FileName, long Size, string Sha256, string SessionKey)
{
public SortedDictionary<int, byte[]> Parts { get; } = [];
public byte[] Bytes => [.. Parts.Values.SelectMany(p => p)];
public int Chunks => Parts.Count;
}
}
}