CI / Build and test (pull_request) Failing after 2m3s
CI / PostgreSQL 16 integration (pull_request) Skipped
CI / PostgreSQL 17 integration (pull_request) Skipped
CI / Docker image (pull_request) Skipped
CI / Windows tests and installer (pull_request) Skipped
CI / YAML check (pull_request) Successful in 4s
The owner renamed the product and moved the repository to the self-hosted Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control, and asked for Actions and a first automatic release with the server package and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md). - Chronos everywhere: a byte-exact rename of contents and paths, then a per-language review of the forms the old name's ending had dictated. - Gitea instead of GitHub: the update check, licence, issue and support links read ChronosReleasesUrl; the update check takes the highest stable version, offers a pre-release build its final release and reports a repository it cannot see as a failed check. EULA, security policy and Gitea issue forms live at the repository root. - Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release, nightly image and performance gates in the .NET SDK container with git checkout. The release cross-compiles the Windows programs, checks their icon, version and DPI manifest, packs installers and zips, publishes the Gitea release and pushes the server image (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows runner (WINDOWS_RUNNER). - New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the docker-exec script, which Gitea's service containers do not allow) and pe-check. BREAKING CHANGE: programs, settings, data folders, file types, environment variables and the Docker image use the new name; nothing is migrated from Titano, which never had a release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
345 lines
20 KiB
C#
345 lines
20 KiB
C#
using System.Collections.Concurrent;
|
|
using System.IO;
|
|
using System.Net;
|
|
using System.Net.Http;
|
|
using System.Net.Http.Json;
|
|
using System.Security.Cryptography;
|
|
using Chronos.App.Import;
|
|
using Chronos.Core.Import;
|
|
using Chronos.Protocol;
|
|
using Chronos.Protocol.Client;
|
|
using Chronos.Testing;
|
|
using static Chronos.App.Tests.Import.ImportTestData;
|
|
|
|
namespace Chronos.App.Tests.Import;
|
|
|
|
/// <summary>The "Server" destination over the real protocol client against an in-memory upload endpoint.</summary>
|
|
[Category("import")]
|
|
public sealed class ServerImportSinkTests(TestContext context)
|
|
{
|
|
[Test]
|
|
public async Task EveryPhotoIsUploadedInChunksUnderItsCardNameAndVerifiedByHash()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start, Start.AddHours(2));
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var server = new FakeUploadServer(chunkSize: 256);
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
plan = plan with { Sessions = [plan.Sessions[0] with { Title = "Harbour sunset" }, .. plan.Sessions.Skip(1)] };
|
|
|
|
var outcome = await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation);
|
|
|
|
Assert.Equal(16, outcome.Copied);
|
|
Assert.True(outcome.IsComplete);
|
|
Assert.Equal(16, server.Committed.Count);
|
|
foreach (var item in plan.Items)
|
|
{
|
|
var bytes = File.ReadAllBytes(Path.Combine(card, item.SourcePath));
|
|
var upload = server.Committed.Single(u => u.FileName == Path.GetFileName(item.SourcePath));
|
|
Assert.BytesEqual(bytes, upload.Bytes);
|
|
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(bytes)), upload.Sha256);
|
|
Assert.True(upload.Chunks > 1, "files larger than a chunk travel in several chunks");
|
|
}
|
|
Assert.SequenceEqual(plan.Sessions.Select(s => s.SessionKey).Order(StringComparer.Ordinal),
|
|
server.Committed.Select(u => u.SessionKey).Distinct().Order(StringComparer.Ordinal));
|
|
Assert.Equal("Harbour sunset", server.Titles.Single().Value);
|
|
Assert.Empty(Directory.EnumerateFileSystemEntries(staging));
|
|
}
|
|
|
|
[Test]
|
|
public async Task PhotosTheServerAlreadyHoldsAreNotSentAgain()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start);
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var server = new FakeUploadServer(chunkSize: 4096);
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var destination = new UnindexedDestination(ImportDestinationKind.Server, staging);
|
|
var plan = await destination.PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation);
|
|
var chunks = server.ChunkCount;
|
|
|
|
var again = await LocalImport.RunAsync(plan, new ServerImportSink(client, staging), "EOS R5 · card A", verify: true, null, context.Cancellation);
|
|
Assert.Equal(0, again.Copied);
|
|
Assert.Equal(8, again.Duplicates);
|
|
Assert.Equal(chunks, server.ChunkCount);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFileTheServerRefusesIsListedAndTheImportGoesOn()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start);
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var server = new FakeUploadServer(chunkSize: 4096) { CorruptFile = "IMG_0003.JPG" };
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
var sink = new ServerImportSink(client, staging);
|
|
|
|
var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report);
|
|
Assert.Equal(7, outcome.Copied);
|
|
Assert.Equal(7, server.Committed.Count);
|
|
Assert.Equal(new ImportFailure("DCIM/100CANON/IMG_0003.JPG", "Hash mismatch"), outcome.Failures.Single());
|
|
Assert.Empty(Directory.EnumerateFileSystemEntries(staging));
|
|
}
|
|
|
|
[Test]
|
|
public async Task AFileAProxyRefusesIsListedWithTheCatalogsSentence()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start);
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var server = new FakeUploadServer(chunkSize: 4096) { ProxyRefusesFile = "IMG_0005.JPG" };
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
var sink = new ServerImportSink(client, staging);
|
|
|
|
var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report);
|
|
Assert.Equal(7, outcome.Copied);
|
|
// The proxy's page says "Request Entity Too Large" in English: the photo reads the catalog's sentence instead.
|
|
Assert.Equal(new ImportFailure("DCIM/100CANON/IMG_0005.JPG", Chronos.Core.Localization.Loc.L("Server.Error.TooLarge")), outcome.Failures.Single());
|
|
}
|
|
|
|
[Test]
|
|
public async Task ALostSignInStopsTheImportAndKeepsWhatWasUploaded()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start);
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var server = new FakeUploadServer(chunkSize: 4096) { SignedOutAfter = 3 };
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
var sink = new ServerImportSink(client, staging);
|
|
|
|
var outcome = await LocalImport.RunAsync(plan, sink, "EOS R5 · card A", true, null, context.Cancellation, sink.Report);
|
|
Assert.Equal(3, outcome.Copied, "the uploads the server committed before the sign-in was lost");
|
|
Assert.Equal(5, outcome.Failures.Count);
|
|
Assert.Equal("Sign in again", outcome.Failures[0].Message, "the server's sentence");
|
|
Assert.False(outcome.IsComplete);
|
|
}
|
|
|
|
[Test]
|
|
public async Task TheQuotaIsTheRoomOfAServerImport()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start);
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging) { ServerSpace = 1_000 }
|
|
.PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
Assert.Equal(1_000L, plan.AvailableBytes);
|
|
Assert.False(ImportPlans.Totals(plan, verify: true).HasEnoughSpace, "eight photos do not fit in 1 KB of quota");
|
|
|
|
var unlimited = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null,
|
|
context.Cancellation);
|
|
Assert.Null(unlimited.AvailableBytes);
|
|
Assert.True(ImportPlans.Totals(unlimited, verify: true).HasEnoughSpace);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnImportIntoTheServerIsReportedAsCardEventsFromStartToFinish()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start);
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var server = new FakeUploadServer(chunkSize: 4096);
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
var progress = new List<ImportProgress>();
|
|
|
|
var outcome = await new ServerImportSink(client, staging).RunAsync(plan, "EOS R5 · card A", true, new SyncProgress<ImportProgress>(progress.Add),
|
|
context.Cancellation);
|
|
|
|
Assert.True(outcome.IsComplete);
|
|
Assert.NotEmpty(progress);
|
|
var reports = server.Cards.ToList();
|
|
Assert.Equal(CardImportStates.Started, reports[0].State);
|
|
Assert.Equal((CardImportStates.Finished, plan.FileCount, plan.FileCount, 0), (reports[^1].State, reports[^1].Done, reports[^1].Total, reports[^1].Failed));
|
|
Assert.True(reports.All(r => r.ImportId == plan.Id && r.Card == "EOS R5 · card A"), "every report names the import");
|
|
Assert.True(reports.Skip(1).SkipLast(1).All(r => r.State == CardImportStates.Progress), "progress in between");
|
|
}
|
|
|
|
[Test]
|
|
public async Task AServerWithoutCardEventsStillTakesTheImport()
|
|
{
|
|
var card = Card(context.CreateTempDirectory("card"), 8, Start);
|
|
var staging = context.CreateTempDirectory("staging");
|
|
var server = new FakeUploadServer(chunkSize: 4096) { CardsUnsupported = true };
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var plan = await new UnindexedDestination(ImportDestinationKind.Server, staging).PlanAsync(card, new ImportChoices().ToOptions(), null, null, context.Cancellation);
|
|
|
|
var outcome = await new ServerImportSink(client, staging).RunAsync(plan, "EOS R5 · card A", true, null, context.Cancellation);
|
|
|
|
Assert.Equal(8, outcome.Copied);
|
|
Assert.Equal(1, server.CardAttempts, "a server that answers 404 gets no further reports");
|
|
}
|
|
|
|
[Test]
|
|
public async Task ReportsWaitingForASlowServerAreReplacedByNewerOnes()
|
|
{
|
|
var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
var server = new FakeUploadServer(chunkSize: 4096) { CardGate = gate.Task };
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var time = new ManualTime();
|
|
var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100, time);
|
|
|
|
reporter.Started();
|
|
await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
reporter.Report(Progress(10));
|
|
reporter.Report(Progress(20));
|
|
time.Advance(TimeSpan.FromSeconds(1));
|
|
reporter.Report(Progress(30));
|
|
var finished = reporter.FinishedAsync(null);
|
|
gate.SetResult();
|
|
await finished.WaitAsync(TimeSpan.FromSeconds(10));
|
|
|
|
// The start was on its way; progress 10 waited and was replaced by 30 (20 came within the second), then by the end.
|
|
Assert.SequenceEqual([(CardImportStates.Started, 0), (CardImportStates.Finished, 30)], server.Cards.Select(r => (r.State, r.Done)));
|
|
// Nothing follows the end, not even progress a second later.
|
|
time.Advance(TimeSpan.FromSeconds(1));
|
|
reporter.Report(Progress(40));
|
|
await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(10));
|
|
Assert.Equal(2, server.CardAttempts);
|
|
}
|
|
|
|
[Test]
|
|
public async Task AnImportOverDoesNotWaitForAServerThatDoesNotAnswer()
|
|
{
|
|
var server = new FakeUploadServer(chunkSize: 4096) { CardGate = new TaskCompletionSource().Task };
|
|
using var client = new ChronosClient(new Uri("https://nas.local:8443/"), new CertificatePinning([]), "tests", server) { Token = "t" };
|
|
var reporter = new CardImportReporter(client, Guid.CreateVersion7(), "EOS R5 · card A", 100);
|
|
|
|
reporter.Started();
|
|
await server.CardArrived.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
|
var clock = System.Diagnostics.Stopwatch.StartNew();
|
|
await reporter.FinishedAsync(null).WaitAsync(TimeSpan.FromSeconds(8));
|
|
|
|
// The start hangs (each report would wait 10 s): the end gives both up after two seconds.
|
|
Assert.True(clock.Elapsed < TimeSpan.FromSeconds(6), $"finished after {clock.Elapsed}");
|
|
Assert.Empty(server.Cards);
|
|
for (var i = 0; i < 25 && server.CardAttempts < 2; i++) await Task.Delay(20);
|
|
Assert.Equal(1, server.CardAttempts, "the end was never sent once given up");
|
|
}
|
|
|
|
private static ImportProgress Progress(int done) => new(done, 100, done * 10L, 1000, done, 0, 0, null);
|
|
|
|
/// <summary>Reports synchronously (Progress<T> posts and may arrive late).</summary>
|
|
private sealed class SyncProgress<T>(Action<T> report) : IProgress<T>
|
|
{
|
|
public void Report(T value) => report(value);
|
|
}
|
|
|
|
private sealed class ManualTime : TimeProvider
|
|
{
|
|
private long _ticks;
|
|
|
|
public override long TimestampFrequency => TimeSpan.TicksPerSecond;
|
|
|
|
public override long GetTimestamp() => Interlocked.Read(ref _ticks);
|
|
|
|
public void Advance(TimeSpan by) => Interlocked.Add(ref _ticks, by.Ticks);
|
|
}
|
|
|
|
/// <summary>Start, chunk and complete of BRIEF §14.4, plus the title update and card reports; the hash is checked at completion.</summary>
|
|
private sealed class FakeUploadServer(int chunkSize) : HttpMessageHandler
|
|
{
|
|
private readonly Dictionary<Guid, Upload> _open = [];
|
|
private readonly Dictionary<string, Guid> _sequences = new(StringComparer.Ordinal);
|
|
private readonly Dictionary<string, Guid> _frames = new(StringComparer.Ordinal);
|
|
|
|
public List<Upload> Committed { get; } = [];
|
|
public Dictionary<Guid, string> Titles { get; } = [];
|
|
public int ChunkCount { get; private set; }
|
|
|
|
/// <summary>Flip a byte of every chunk of this file (a damaged transfer).</summary>
|
|
public string? CorruptFile { get; init; }
|
|
|
|
/// <summary>A reverse proxy in front of the server refuses the chunks of this file: 413 with its own HTML page.</summary>
|
|
public string? ProxyRefusesFile { get; init; }
|
|
|
|
/// <summary>Answer 401 to new uploads once this many were committed (the sign-in expired).</summary>
|
|
public int? SignedOutAfter { get; init; }
|
|
|
|
/// <summary>Card reports taken, in order (they arrive on another thread than the uploads).</summary>
|
|
public ConcurrentQueue<CardImportReport> Cards { get; } = new();
|
|
|
|
public int CardAttempts => _cardAttempts;
|
|
|
|
/// <summary>A server from before card events: 404.</summary>
|
|
public bool CardsUnsupported { get; init; }
|
|
|
|
/// <summary>Card reports wait for this (a slow server); <see cref="CardArrived"/> says when the first one is waiting.</summary>
|
|
public Task? CardGate { get; init; }
|
|
|
|
public TaskCompletionSource CardArrived { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
private int _cardAttempts;
|
|
|
|
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
|
{
|
|
var path = request.RequestUri!.AbsolutePath.Trim('/').Split('/');
|
|
if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads", "cards"])
|
|
{
|
|
Interlocked.Increment(ref _cardAttempts);
|
|
if (CardsUnsupported) return new HttpResponseMessage(HttpStatusCode.NotFound);
|
|
var report = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.CardImportReport, cancellationToken))!;
|
|
CardArrived.TrySetResult();
|
|
if (CardGate is { } gate) await gate.WaitAsync(cancellationToken);
|
|
Cards.Enqueue(report);
|
|
return new HttpResponseMessage(HttpStatusCode.NoContent);
|
|
}
|
|
if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads"])
|
|
{
|
|
var start = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.StartUploadRequest, cancellationToken))!;
|
|
if (Committed.Count >= SignedOutAfter)
|
|
return Json(new ApiError("unauthorized", "Sign in again"), ProtocolJson.Default.ApiError, HttpStatusCode.Unauthorized);
|
|
if (_frames.TryGetValue(start.Sha256, out var existing))
|
|
return Json(new StartUploadResponse(null, chunkSize, [], true, existing), ProtocolJson.Default.StartUploadResponse);
|
|
var upload = new Upload(Guid.NewGuid(), start.FileName, start.Size, start.Sha256, start.SessionKey ?? "user");
|
|
_open[upload.Id] = upload;
|
|
return Json(new StartUploadResponse(upload.Id, chunkSize, [], false, null), ProtocolJson.Default.StartUploadResponse);
|
|
}
|
|
if (request.Method == HttpMethod.Put && path is ["api", "v1", "uploads", var id, "chunks", var n])
|
|
{
|
|
var upload = _open[Guid.Parse(id)];
|
|
if (upload.FileName == ProxyRefusesFile)
|
|
{
|
|
return new HttpResponseMessage(HttpStatusCode.RequestEntityTooLarge)
|
|
{
|
|
Content = new StringContent("<html><body><h1>413 Request Entity Too Large</h1></body></html>", System.Text.Encoding.UTF8, "text/html"),
|
|
};
|
|
}
|
|
var bytes = await request.Content!.ReadAsByteArrayAsync(cancellationToken);
|
|
if (upload.FileName == CorruptFile) bytes[0] ^= 0xFF;
|
|
upload.Parts[int.Parse(n, System.Globalization.CultureInfo.InvariantCulture)] = bytes;
|
|
ChunkCount++;
|
|
return Json(new ChunkReceived(int.Parse(n, System.Globalization.CultureInfo.InvariantCulture), [.. upload.Parts.Keys]), ProtocolJson.Default.ChunkReceived);
|
|
}
|
|
if (request.Method == HttpMethod.Post && path is ["api", "v1", "uploads", var done, "complete"])
|
|
{
|
|
var upload = _open[Guid.Parse(done)];
|
|
if (Convert.ToHexStringLower(SHA256.HashData(upload.Bytes)) != upload.Sha256)
|
|
return Json(new ApiError("hash_mismatch", "Hash mismatch"), ProtocolJson.Default.ApiError, HttpStatusCode.UnprocessableEntity);
|
|
_open.Remove(upload.Id);
|
|
Committed.Add(upload);
|
|
var frame = Guid.NewGuid();
|
|
_frames[upload.Sha256] = frame;
|
|
if (!_sequences.TryGetValue(upload.SessionKey, out var sequence)) _sequences[upload.SessionKey] = sequence = Guid.NewGuid();
|
|
return Json(new CompleteUploadResponse(frame, sequence, Committed.Count(u => u.SessionKey == upload.SessionKey) - 1),
|
|
ProtocolJson.Default.CompleteUploadResponse);
|
|
}
|
|
if (request.Method == HttpMethod.Patch && path is ["api", "v1", "sequences", var sequenceId])
|
|
{
|
|
var update = (await request.Content!.ReadFromJsonAsync(ProtocolJson.Default.UpdateSequenceRequest, cancellationToken))!;
|
|
Titles[Guid.Parse(sequenceId)] = update.Title!;
|
|
return Json(new SequenceDto(Guid.Parse(sequenceId), update.Title, update.Title!, "f", null, null, null, null, null, 0, null, 0, 0, 0, null, [], 0, null,
|
|
1, 1, 0, null, DateTimeOffset.UtcNow), ProtocolJson.Default.SequenceDto);
|
|
}
|
|
return new HttpResponseMessage(HttpStatusCode.NotFound);
|
|
}
|
|
|
|
private static HttpResponseMessage Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> type, HttpStatusCode status = HttpStatusCode.OK) =>
|
|
new(status) { Content = JsonContent.Create(value, type) };
|
|
|
|
public sealed record Upload(Guid Id, string FileName, long Size, string Sha256, string SessionKey)
|
|
{
|
|
public SortedDictionary<int, byte[]> Parts { get; } = [];
|
|
public byte[] Bytes => [.. Parts.Values.SelectMany(p => p)];
|
|
public int Chunks => Parts.Count;
|
|
}
|
|
}
|
|
}
|