diff --git a/src/Titano.Uploader/Uploads/IUploadTransport.cs b/src/Titano.Uploader/Uploads/IUploadTransport.cs new file mode 100644 index 00000000..33298391 --- /dev/null +++ b/src/Titano.Uploader/Uploads/IUploadTransport.cs @@ -0,0 +1,47 @@ +using Titano.Protocol.Client; + +namespace Titano.Uploader.Uploads; + +/// How the server answered a successful upload. +public enum UploadOutcome +{ + /// Committed after the server checked the SHA-256 (CompleteUploadResponse). + Verified, + + /// The server already held this SHA-256 (AlreadyExists). + AlreadyOnServer, +} + +/// What the upload needs from the server connection. +public interface IUploadTransport +{ + /// True while signed in and reachable; the engine waits otherwise. + bool IsReady { get; } + + /// Raised (on the UI thread) when changes. + event EventHandler? ReadyChanged; + + /// + /// Sends one file whose hash is known. Throws when the server cannot be + /// used right now (offline, signed out, certificate changed); other failures concern this file only. + /// + Task<(UploadOutcome Outcome, Guid? FrameId)> UploadAsync(UploadItem item, Guid? sequenceId, string sha256, IProgress progress, CancellationToken ct); + + /// Creates a sequence for a "new sequence" batch. + Task CreateSequenceAsync(string? title, CancellationToken ct); +} + +/// Why the server cannot be used. +public enum ServerProblem +{ + Unreachable, + SignedOut, + CertificateChanged, +} + +/// The server cannot take uploads right now; the item goes back in line without spending a retry. +public sealed class ServerUnavailableException(ServerProblem problem, Exception? inner = null) + : Exception("Server unavailable: " + problem, inner) +{ + public ServerProblem Problem { get; } = problem; +} diff --git a/src/Titano.Uploader/Uploads/UploadEngine.cs b/src/Titano.Uploader/Uploads/UploadEngine.cs new file mode 100644 index 00000000..a6f26697 --- /dev/null +++ b/src/Titano.Uploader/Uploads/UploadEngine.cs @@ -0,0 +1,365 @@ +using System.ComponentModel; +using System.IO; +using Titano.Core.Diagnostics; +using Titano.Protocol.Client; + +namespace Titano.Uploader.Uploads; + +/// What the upload engine is doing. +public enum EngineStatus +{ + Idle, + Running, + Paused, + WaitingForServer, +} + +/// Queue control used by the window and the tray. +public interface IUploadEngine : INotifyPropertyChanged +{ + EngineStatus Status { get; } + + /// Current throughput over the last seconds. + double BytesPerSecond { get; } + + /// Raised when the last pending item finished (balloon on completion). + event EventHandler? Drained; + + /// Starts work for newly queued items without waiting for the next tick. + void Pump(); + + void Pause(); + void Resume(); + + /// Stops every transfer and clears the queue; files already uploaded stay on the server. + void StopAndClear(); + + /// Puts a failed item back in line with a fresh retry budget. + void Retry(UploadItem item); +} + +/// +/// Works through the with concurrent files: hash, upload +/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source the server confirmed. +/// Lives on the UI thread: state changes happen there while hashing and HTTP run on the thread pool. +/// +public sealed class UploadEngine : IUploadEngine, IDisposable +{ + public const int Parallelism = 2; + + private readonly UploadQueue _queue; + private readonly IUploadTransport _transport; + private readonly Func _deleteAfterUpload; + private readonly TimeProvider _time; + private readonly long _origin; + private readonly RateMeter _meter = new(); + private readonly Dictionary _running = []; + private readonly Dictionary> _creating = []; + private readonly CancellationTokenSource _lifetime = new(); + private EngineStatus _status; + private double _bytesPerSecond; + private bool _hadWork; + + public UploadEngine(UploadQueue queue, IUploadTransport transport, Func deleteAfterUpload, TimeProvider? time = null) + { + _queue = queue; + _transport = transport; + _deleteAfterUpload = deleteAfterUpload; + _time = time ?? TimeProvider.System; + _origin = _time.GetTimestamp(); + _transport.ReadyChanged += (_, _) => Pump(); + } + + public event PropertyChangedEventHandler? PropertyChanged; + + public event EventHandler? Drained; + + /// Raised when the server confirmed an item (verified or already present). + public event EventHandler? ItemConfirmed; + + public EngineStatus Status + { + get => _status; + private set + { + if (_status == value) return; + _status = value; + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Status))); + } + } + + public double BytesPerSecond + { + get => _bytesPerSecond; + private set + { + if (_bytesPerSecond.Equals(value)) return; + _bytesPerSecond = value; + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(BytesPerSecond))); + } + } + + private TimeSpan Now => _time.GetElapsedTime(_origin); + private DateTime UtcNow => _time.GetUtcNow().UtcDateTime; + + /// Starts the once-a-second pump on the calling (UI) thread. + public void Start() => _ = TickAsync(_lifetime.Token); + + /// Starts work for newly queued items without waiting for the next tick. + public void Pump() + { + if (_lifetime.IsCancellationRequested) return; + while (!_queue.IsPaused && _transport.IsReady && _running.Count < Parallelism) + { + var item = _queue.NextReady(UtcNow); + if (item is null) break; + Begin(item); + } + Refresh(); + } + + public void Pause() + { + _queue.IsPaused = true; + CancelRunning(); + Refresh(); + } + + public void Resume() + { + _queue.IsPaused = false; + Pump(); + } + + public void StopAndClear() + { + CancelRunning(); + _queue.Clear(); + _creating.Clear(); + _meter.Reset(); + _hadWork = false; + Refresh(); + } + + public void Retry(UploadItem item) + { + _queue.Requeue(item); + Pump(); + } + + public void Dispose() + { + _lifetime.Cancel(); + CancelRunning(); + _lifetime.Dispose(); + } + + private async Task TickAsync(CancellationToken ct) + { + using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1), _time); + try + { + do + { + Pump(); + } + while (await timer.WaitForNextTickAsync(ct)); + } + catch (OperationCanceledException) + { + } + } + + private void Begin(UploadItem item) + { + if (item.State == UploadItemState.Failed) + { + item.Retries++; + item.NextRetryUtc = null; + item.Failure = UploadFailure.None; + item.FailureDetail = null; + } + // Leaving Queued/Failed synchronously keeps NextReady from handing the item out twice. + item.State = UploadItemState.Hashing; + var cts = CancellationTokenSource.CreateLinkedTokenSource(_lifetime.Token); + _running[item] = cts; + _hadWork = true; + _ = RunAsync(item, cts); + } + + private async Task RunAsync(UploadItem item, CancellationTokenSource cts) + { + var ct = cts.Token; + try + { + await HashIfNeededAsync(item, ct); + if (item.State == UploadItemState.Failed) return; + + item.State = UploadItemState.Uploading; + item.BytesSent = 0; + var batch = _queue.GetBatch(item.BatchId); + Guid? sequenceId = batch?.Mode switch + { + UploadTargetMode.Sequence => batch.SequenceId, + UploadTargetMode.NewSequence => await EnsureSequenceAsync(batch, ct), + _ => null, + }; + + var baseline = true; + var progress = new Progress(p => + { + if (item.State != UploadItemState.Uploading) return; + // The first report of a resumed upload counts chunks the server already had: not throughput. + if (!baseline) _meter.Add(Math.Max(0, p.BytesSent - item.BytesSent), Now); + baseline = false; + item.BytesSent = p.BytesSent; + }); + var (outcome, frameId) = await _transport.UploadAsync(item, sequenceId, item.Sha256!, progress, ct); + + item.FrameId = frameId; + item.BytesSent = item.Size; + item.State = outcome == UploadOutcome.Verified ? UploadItemState.Verified : UploadItemState.Skipped; + _queue.OnChanged(); + ItemConfirmed?.Invoke(this, item); + if (_deleteAfterUpload()) DeleteConfirmedSource(item); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + Requeue(item); + } + catch (ServerUnavailableException ex) + { + Log.Info("upload", $"{item.FileName} waits for the server ({ex.Problem})"); + Requeue(item); + } + catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException) + { + _queue.Fail(item, UploadFailure.FileMissing, null, UtcNow); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("upload", $"Cannot read {item.Path}", ex); + _queue.Fail(item, File.Exists(item.Path) ? UploadFailure.FileUnreadable : UploadFailure.FileMissing, null, UtcNow); + } + catch (TitanoApiException ex) + { + Log.Warning("upload", $"Server refused {item.Path}: {ex.Error.Code}", ex); + _queue.Fail(item, UploadFailure.ServerRejected, ex.Error.Message, UtcNow); + } + catch (Exception ex) + { + // Worker boundary: nothing may leave an item stuck in an active state. + Log.Error("upload", $"Upload of {item.Path} failed", ex); + _queue.Fail(item, UploadFailure.Unexpected, null, UtcNow); + } + finally + { + _running.Remove(item); + cts.Dispose(); + Pump(); + } + } + + /// Hashes when the file is new or changed; a file that changes while being read fails for a later retry. + private async Task HashIfNeededAsync(UploadItem item, CancellationToken ct) + { + var file = new FileInfo(item.Path); + if (!file.Exists) + { + _queue.Fail(item, UploadFailure.FileMissing, null, UtcNow); + return; + } + if (file.Length != item.Size || file.LastWriteTimeUtc != item.ModifiedUtc) + { + item.Sha256 = null; + item.Size = file.Length; + item.ModifiedUtc = file.LastWriteTimeUtc; + } + if (item.Sha256 is not null) return; + + var sha = await Task.Run(() => TitanoClient.HashFileAsync(item.Path, ct), ct); + file.Refresh(); + if (!file.Exists || file.Length != item.Size || file.LastWriteTimeUtc != item.ModifiedUtc) + { + _queue.Fail(item, UploadFailure.FileChanged, null, UtcNow); + return; + } + item.Sha256 = sha; + _queue.OnChanged(); + } + + /// Creates the batch's sequence once, shared by concurrent items of the same batch. + private async Task EnsureSequenceAsync(UploadBatch batch, CancellationToken ct) + { + if (batch.SequenceId is { } known) return known; + if (!_creating.TryGetValue(batch.Id, out var creation)) + { + creation = _transport.CreateSequenceAsync(batch.Title, _lifetime.Token); + _creating[batch.Id] = creation; + } + try + { + var id = await creation.WaitAsync(ct); + batch.SequenceId = id; + _queue.OnChanged(); + return id; + } + catch when (creation.IsFaulted || creation.IsCanceled) + { + _creating.Remove(batch.Id); + throw; + } + } + + /// + /// BRIEF ยง0.2 rule 8: deletes only a file the server confirmed by hash, and only when it is still exactly the + /// bytes that were hashed (same size and write time). + /// + private static void DeleteConfirmedSource(UploadItem item) + { + try + { + var file = new FileInfo(item.Path); + if (!file.Exists) return; + if (file.Length != item.Size || file.LastWriteTimeUtc != item.ModifiedUtc) + { + Log.Warning("upload", $"Kept {item.Path}: it changed after the server verified it"); + return; + } + file.Delete(); + Log.Info("upload", $"Deleted {item.Path} after the server confirmed SHA-256 {item.Sha256}"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("upload", $"Could not delete {item.Path}", ex); + } + } + + private void Requeue(UploadItem item) + { + if (!item.IsActive) return; + item.State = UploadItemState.Queued; + item.BytesSent = 0; + } + + private void CancelRunning() + { + foreach (var cts in _running.Values.ToList()) cts.Cancel(); + } + + private void Refresh() + { + BytesPerSecond = _meter.BytesPerSecond(Now); + var pending = _queue.HasPending; + Status = _queue.IsPaused && pending ? EngineStatus.Paused + : pending && !_transport.IsReady ? EngineStatus.WaitingForServer + : pending || _running.Count > 0 ? EngineStatus.Running + : EngineStatus.Idle; + if (_hadWork && !pending && _running.Count == 0) + { + _hadWork = false; + _meter.Reset(); + Drained?.Invoke(this, EventArgs.Empty); + } + } +} diff --git a/tests/Titano.App.Tests/Uploader/SingleThreadedContext.cs b/tests/Titano.App.Tests/Uploader/SingleThreadedContext.cs new file mode 100644 index 00000000..c1d1c46c --- /dev/null +++ b/tests/Titano.App.Tests/Uploader/SingleThreadedContext.cs @@ -0,0 +1,41 @@ +using System.Collections.Concurrent; + +namespace Titano.App.Tests.Uploader; + +/// Runs async test code on one thread, as the UI dispatcher does for the upload engine. +internal sealed class SingleThreadedContext : SynchronizationContext +{ + private readonly BlockingCollection<(SendOrPostCallback Callback, object? State)> _queue = []; + + public override void Post(SendOrPostCallback d, object? state) + { + try + { + _queue.Add((d, state)); + } + catch (InvalidOperationException) + { + // Late progress reports after the test body finished are dropped. + } + } + + public override void Send(SendOrPostCallback d, object? state) => throw new NotSupportedException(); + + public static void Run(Func body) + { + var previous = Current; + var context = new SingleThreadedContext(); + SetSynchronizationContext(context); + try + { + var task = body(); + task.ContinueWith(_ => context._queue.CompleteAdding(), TaskScheduler.Default); + foreach (var (callback, state) in context._queue.GetConsumingEnumerable()) callback(state); + task.GetAwaiter().GetResult(); + } + finally + { + SetSynchronizationContext(previous); + } + } +} diff --git a/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs b/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs new file mode 100644 index 00000000..550f6b5f --- /dev/null +++ b/tests/Titano.App.Tests/Uploader/UploadEngineTests.cs @@ -0,0 +1,223 @@ +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Security.Cryptography; +using Titano.Protocol; +using Titano.Protocol.Client; +using Titano.Testing; +using Titano.Uploader.Uploads; + +namespace Titano.App.Tests.Uploader; + +public sealed class UploadEngineTests(TestContext context) +{ + private readonly ManualTime _time = new(); + + [Test] + public void VerifiedUploadsKeepTheSourceByDefault() => SingleThreadedContext.Run(async () => + { + var queue = Queue(UploadTargetMode.Sessions, 2); + var transport = new FakeTransport(); + using var engine = new UploadEngine(queue, transport, () => false, _time); + var confirmed = new List(); + engine.ItemConfirmed += (_, item) => confirmed.Add(item); + + engine.Pump(); + await Until(() => queue.Items.All(i => i.IsDone)); + + Assert.True(queue.Items.All(i => i.State == UploadItemState.Verified && i.FrameId is not null)); + Assert.True(queue.Items.All(i => File.Exists(i.Path))); + Assert.Equal(Sha256(queue.Items[0].Path), transport.Calls[0].Sha256); + Assert.Equal(2, confirmed.Count); + }); + + [Test] + public void DeleteAfterUploadRemovesOnlyConfirmedUnchangedFiles() => SingleThreadedContext.Run(async () => + { + var queue = Queue(UploadTargetMode.Sessions, 3); + var (verified, existing, changed) = (queue.Items[0], queue.Items[1], queue.Items[2]); + var transport = new FakeTransport + { + Respond = item => + { + if (item == existing) return (UploadOutcome.AlreadyOnServer, null); + // Written to after it was hashed: the bytes on disk are no longer the verified ones. + if (item == changed) File.AppendAllText(item.Path, "more"); + return (UploadOutcome.Verified, Guid.CreateVersion7()); + }, + }; + using var engine = new UploadEngine(queue, transport, () => true, _time); + + engine.Pump(); + await Until(() => queue.Items.All(i => i.IsDone)); + + Assert.Equal(UploadItemState.Skipped, existing.State); + Assert.False(File.Exists(verified.Path)); + Assert.False(File.Exists(existing.Path)); + Assert.True(File.Exists(changed.Path), "a file changed after verification is kept"); + }); + + [Test] + public void RejectedFileIsRetriedAfterThirtySeconds() => SingleThreadedContext.Run(async () => + { + var queue = Queue(UploadTargetMode.Sessions, 1); + var item = queue.Items[0]; + var attempts = 0; + var transport = new FakeTransport + { + Respond = _ => ++attempts == 1 + ? throw new TitanoApiException(HttpStatusCode.UnprocessableEntity, new ApiError("hash_mismatch", "Hash mismatch")) + : (UploadOutcome.Verified, Guid.CreateVersion7()), + }; + using var engine = new UploadEngine(queue, transport, () => false, _time); + + engine.Pump(); + await Until(() => item.State == UploadItemState.Failed); + Assert.Equal(UploadFailure.ServerRejected, item.Failure); + Assert.Equal("Hash mismatch", item.FailureDetail); + Assert.Equal(_time.UtcNow + UploadQueue.RetryDelay, item.NextRetryUtc); + + _time.Advance(TimeSpan.FromSeconds(29)); + engine.Pump(); + Assert.Equal(UploadItemState.Failed, item.State); + + _time.Advance(TimeSpan.FromSeconds(1)); + engine.Pump(); + await Until(() => item.IsDone); + Assert.Equal(1, item.Retries); + Assert.Equal(UploadFailure.None, item.Failure); + }); + + [Test] + public void UnreachableServerRequeuesWithoutSpendingARetry() => SingleThreadedContext.Run(async () => + { + var queue = Queue(UploadTargetMode.Sessions, 1); + var item = queue.Items[0]; + var transport = new FakeTransport(); + transport.Respond = _ => + { + transport.SetReady(false); + throw new ServerUnavailableException(ServerProblem.Unreachable); + }; + using var engine = new UploadEngine(queue, transport, () => false, _time); + + engine.Pump(); + await Until(() => item.State == UploadItemState.Queued && transport.Calls.Count == 1); + Assert.Equal(0, item.Retries); + Assert.NotNull(item.Sha256); + Assert.Equal(EngineStatus.WaitingForServer, engine.Status); + + transport.Respond = _ => (UploadOutcome.Verified, Guid.CreateVersion7()); + transport.SetReady(true); + await Until(() => item.IsDone); + }); + + [Test] + public void NewSequenceIsCreatedOnceForTheBatch() => SingleThreadedContext.Run(async () => + { + var queue = Queue(UploadTargetMode.NewSequence, 3); + var transport = new FakeTransport(); + using var engine = new UploadEngine(queue, transport, () => false, _time); + + engine.Pump(); + await Until(() => queue.Items.All(i => i.IsDone)); + + Assert.Equal(1, transport.SequencesCreated); + Assert.True(transport.Calls.All(c => c.SequenceId == FakeTransport.CreatedSequence)); + Assert.Equal(FakeTransport.CreatedSequence, queue.Batches.Single().SequenceId); + }); + + [Test] + public void AtMostTwoFilesAreInFlightAndPauseRequeuesThem() => SingleThreadedContext.Run(async () => + { + var queue = Queue(UploadTargetMode.Sessions, 4); + var transport = new FakeTransport { Block = true }; + using var engine = new UploadEngine(queue, transport, () => false, _time); + + engine.Pump(); + await Until(() => queue.Items.Count(i => i.State == UploadItemState.Uploading) == UploadEngine.Parallelism); + Assert.Equal(UploadEngine.Parallelism, queue.Items.Count(i => i.IsActive)); + + engine.Pause(); + await Until(() => queue.Items.All(i => i.State == UploadItemState.Queued)); + Assert.Equal(EngineStatus.Paused, engine.Status); + + transport.Block = false; + engine.Resume(); + await Until(() => queue.Items.All(i => i.IsDone)); + Assert.False(queue.IsPaused); + }); + + private UploadQueue Queue(UploadTargetMode mode, int count) + { + var dir = context.CreateTempDirectory(); + var queue = new UploadQueue(); + var entries = Enumerable.Range(0, count).Select(i => + { + var path = Path.Combine(dir, $"IMG_{i:0000}.JPG"); + File.WriteAllBytes(path, RandomNumberGenerator.GetBytes(4096 + i)); + var info = new FileInfo(path); + return new QueueEntry(path, info.Length, info.LastWriteTimeUtc, info.LastWriteTime, "k", null); + }).ToList(); + queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = mode, Title = mode == UploadTargetMode.NewSequence ? "Roof" : null }, entries); + return queue; + } + + private static string Sha256(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path))); + + private static async Task Until(Func condition) + { + var clock = Stopwatch.StartNew(); + while (!condition()) + { + if (clock.Elapsed > TimeSpan.FromSeconds(15)) throw new TimeoutException("Condition not reached"); + await Task.Delay(5); + } + } + + private sealed class ManualTime : TimeProvider + { + private DateTimeOffset _now = new(2026, 9, 28, 18, 0, 0, TimeSpan.Zero); + + public DateTime UtcNow => _now.UtcDateTime; + + public void Advance(TimeSpan by) => _now += by; + + public override DateTimeOffset GetUtcNow() => _now; + } + + private sealed class FakeTransport : IUploadTransport + { + public static readonly Guid CreatedSequence = Guid.CreateVersion7(); + + public event EventHandler? ReadyChanged; + + public bool IsReady { get; private set; } = true; + public bool Block { get; set; } + public int SequencesCreated { get; private set; } + public List<(string Path, Guid? SequenceId, string Sha256)> Calls { get; } = []; + public Func Respond { get; set; } = _ => (UploadOutcome.Verified, Guid.CreateVersion7()); + + public void SetReady(bool ready) + { + IsReady = ready; + ReadyChanged?.Invoke(this, EventArgs.Empty); + } + + public async Task<(UploadOutcome Outcome, Guid? FrameId)> UploadAsync(UploadItem item, Guid? sequenceId, string sha256, IProgress progress, CancellationToken ct) + { + Calls.Add((item.Path, sequenceId, sha256)); + progress.Report(new UploadProgress(0, item.Size)); + while (Block) await Task.Delay(5, ct); + await Task.Yield(); + progress.Report(new UploadProgress(item.Size, item.Size)); + return Respond(item); + } + + public Task CreateSequenceAsync(string? title, CancellationToken ct) + { + SequencesCreated++; + return Task.FromResult(CreatedSequence); + } + } +}