feat(uploader): upload engine with queue retries and verified-only source deletion

Two parallel files: hash, resumable upload through a transport, then
verified or already-on-server. Failed items retry after 30 s up to 3
times on top of the client's transient retries; an unreachable server
requeues without spending a retry. A new-sequence batch creates its
sequence once. "Delete after upload" removes only files the server
confirmed and that still match the hashed size and write time.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 10:27:24 +00:00
1 parent 62c7b0d956
commit 2a74c12fd1
4 files changed
+676

No files matched your search

@@ -0,0 +1,47 @@
using Titano.Protocol.Client;
namespace Titano.Uploader.Uploads;
/// <summary>How the server answered a successful upload.</summary>
public enum UploadOutcome
{
/// <summary>Committed after the server checked the SHA-256 (<c>CompleteUploadResponse</c>).</summary>
Verified,
/// <summary>The server already held this SHA-256 (<c>AlreadyExists</c>).</summary>
AlreadyOnServer,
}
/// <summary>What the upload needs from the server connection.</summary>
public interface IUploadTransport
{
/// <summary>True while signed in and reachable; the engine waits otherwise.</summary>
bool IsReady { get; }
/// <summary>Raised (on the UI thread) when <see cref="IsReady"/> changes.</summary>
event EventHandler? ReadyChanged;
/// <summary>
/// Sends one file whose hash is known. Throws <see cref="ServerUnavailableException"/> when the server cannot be
/// used right now (offline, signed out, certificate changed); other failures concern this file only.
/// </summary>
Task<(UploadOutcome Outcome, Guid? FrameId)> UploadAsync(UploadItem item, Guid? sequenceId, string sha256, IProgress<UploadProgress> progress, CancellationToken ct);
/// <summary>Creates a sequence for a "new sequence" batch.</summary>
Task<Guid> CreateSequenceAsync(string? title, CancellationToken ct);
}
/// <summary>Why the server cannot be used.</summary>
public enum ServerProblem
{
Unreachable,
SignedOut,
CertificateChanged,
}
/// <summary>The server cannot take uploads right now; the item goes back in line without spending a retry.</summary>
public sealed class ServerUnavailableException(ServerProblem problem, Exception? inner = null)
: Exception("Server unavailable: " + problem, inner)
{
public ServerProblem Problem { get; } = problem;
}
+365
View File
@@ -0,0 +1,365 @@
using System.ComponentModel;
using System.IO;
using Titano.Core.Diagnostics;
using Titano.Protocol.Client;
namespace Titano.Uploader.Uploads;
/// <summary>What the upload engine is doing.</summary>
public enum EngineStatus
{
Idle,
Running,
Paused,
WaitingForServer,
}
/// <summary>Queue control used by the window and the tray.</summary>
public interface IUploadEngine : INotifyPropertyChanged
{
EngineStatus Status { get; }
/// <summary>Current throughput over the last seconds.</summary>
double BytesPerSecond { get; }
/// <summary>Raised when the last pending item finished (balloon on completion).</summary>
event EventHandler? Drained;
/// <summary>Starts work for newly queued items without waiting for the next tick.</summary>
void Pump();
void Pause();
void Resume();
/// <summary>Stops every transfer and clears the queue; files already uploaded stay on the server.</summary>
void StopAndClear();
/// <summary>Puts a failed item back in line with a fresh retry budget.</summary>
void Retry(UploadItem item);
}
/// <summary>
/// Works through the <see cref="UploadQueue"/> with <see cref="Parallelism"/> concurrent files: hash, upload
/// (resumable chunks; the server verifies the SHA-256), then optionally delete the source the server confirmed.
/// Lives on the UI thread: state changes happen there while hashing and HTTP run on the thread pool.
/// </summary>
public sealed class UploadEngine : IUploadEngine, IDisposable
{
public const int Parallelism = 2;
private readonly UploadQueue _queue;
private readonly IUploadTransport _transport;
private readonly Func<bool> _deleteAfterUpload;
private readonly TimeProvider _time;
private readonly long _origin;
private readonly RateMeter _meter = new();
private readonly Dictionary<UploadItem, CancellationTokenSource> _running = [];
private readonly Dictionary<Guid, Task<Guid>> _creating = [];
private readonly CancellationTokenSource _lifetime = new();
private EngineStatus _status;
private double _bytesPerSecond;
private bool _hadWork;
public UploadEngine(UploadQueue queue, IUploadTransport transport, Func<bool> deleteAfterUpload, TimeProvider? time = null)
{
_queue = queue;
_transport = transport;
_deleteAfterUpload = deleteAfterUpload;
_time = time ?? TimeProvider.System;
_origin = _time.GetTimestamp();
_transport.ReadyChanged += (_, _) => Pump();
}
public event PropertyChangedEventHandler? PropertyChanged;
public event EventHandler? Drained;
/// <summary>Raised when the server confirmed an item (verified or already present).</summary>
public event EventHandler<UploadItem>? ItemConfirmed;
public EngineStatus Status
{
get => _status;
private set
{
if (_status == value) return;
_status = value;
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Status)));
}
}
public double BytesPerSecond
{
get => _bytesPerSecond;
private set
{
if (_bytesPerSecond.Equals(value)) return;
_bytesPerSecond = value;
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(BytesPerSecond)));
}
}
private TimeSpan Now => _time.GetElapsedTime(_origin);
private DateTime UtcNow => _time.GetUtcNow().UtcDateTime;
/// <summary>Starts the once-a-second pump on the calling (UI) thread.</summary>
public void Start() => _ = TickAsync(_lifetime.Token);
/// <summary>Starts work for newly queued items without waiting for the next tick.</summary>
public void Pump()
{
if (_lifetime.IsCancellationRequested) return;
while (!_queue.IsPaused && _transport.IsReady && _running.Count < Parallelism)
{
var item = _queue.NextReady(UtcNow);
if (item is null) break;
Begin(item);
}
Refresh();
}
public void Pause()
{
_queue.IsPaused = true;
CancelRunning();
Refresh();
}
public void Resume()
{
_queue.IsPaused = false;
Pump();
}
public void StopAndClear()
{
CancelRunning();
_queue.Clear();
_creating.Clear();
_meter.Reset();
_hadWork = false;
Refresh();
}
public void Retry(UploadItem item)
{
_queue.Requeue(item);
Pump();
}
public void Dispose()
{
_lifetime.Cancel();
CancelRunning();
_lifetime.Dispose();
}
private async Task TickAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(1), _time);
try
{
do
{
Pump();
}
while (await timer.WaitForNextTickAsync(ct));
}
catch (OperationCanceledException)
{
}
}
private void Begin(UploadItem item)
{
if (item.State == UploadItemState.Failed)
{
item.Retries++;
item.NextRetryUtc = null;
item.Failure = UploadFailure.None;
item.FailureDetail = null;
}
// Leaving Queued/Failed synchronously keeps NextReady from handing the item out twice.
item.State = UploadItemState.Hashing;
var cts = CancellationTokenSource.CreateLinkedTokenSource(_lifetime.Token);
_running[item] = cts;
_hadWork = true;
_ = RunAsync(item, cts);
}
private async Task RunAsync(UploadItem item, CancellationTokenSource cts)
{
var ct = cts.Token;
try
{
await HashIfNeededAsync(item, ct);
if (item.State == UploadItemState.Failed) return;
item.State = UploadItemState.Uploading;
item.BytesSent = 0;
var batch = _queue.GetBatch(item.BatchId);
Guid? sequenceId = batch?.Mode switch
{
UploadTargetMode.Sequence => batch.SequenceId,
UploadTargetMode.NewSequence => await EnsureSequenceAsync(batch, ct),
_ => null,
};
var baseline = true;
var progress = new Progress<UploadProgress>(p =>
{
if (item.State != UploadItemState.Uploading) return;
// The first report of a resumed upload counts chunks the server already had: not throughput.
if (!baseline) _meter.Add(Math.Max(0, p.BytesSent - item.BytesSent), Now);
baseline = false;
item.BytesSent = p.BytesSent;
});
var (outcome, frameId) = await _transport.UploadAsync(item, sequenceId, item.Sha256!, progress, ct);
item.FrameId = frameId;
item.BytesSent = item.Size;
item.State = outcome == UploadOutcome.Verified ? UploadItemState.Verified : UploadItemState.Skipped;
_queue.OnChanged();
ItemConfirmed?.Invoke(this, item);
if (_deleteAfterUpload()) DeleteConfirmedSource(item);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
Requeue(item);
}
catch (ServerUnavailableException ex)
{
Log.Info("upload", $"{item.FileName} waits for the server ({ex.Problem})");
Requeue(item);
}
catch (Exception ex) when (ex is FileNotFoundException or DirectoryNotFoundException)
{
_queue.Fail(item, UploadFailure.FileMissing, null, UtcNow);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("upload", $"Cannot read {item.Path}", ex);
_queue.Fail(item, File.Exists(item.Path) ? UploadFailure.FileUnreadable : UploadFailure.FileMissing, null, UtcNow);
}
catch (TitanoApiException ex)
{
Log.Warning("upload", $"Server refused {item.Path}: {ex.Error.Code}", ex);
_queue.Fail(item, UploadFailure.ServerRejected, ex.Error.Message, UtcNow);
}
catch (Exception ex)
{
// Worker boundary: nothing may leave an item stuck in an active state.
Log.Error("upload", $"Upload of {item.Path} failed", ex);
_queue.Fail(item, UploadFailure.Unexpected, null, UtcNow);
}
finally
{
_running.Remove(item);
cts.Dispose();
Pump();
}
}
/// <summary>Hashes when the file is new or changed; a file that changes while being read fails for a later retry.</summary>
private async Task HashIfNeededAsync(UploadItem item, CancellationToken ct)
{
var file = new FileInfo(item.Path);
if (!file.Exists)
{
_queue.Fail(item, UploadFailure.FileMissing, null, UtcNow);
return;
}
if (file.Length != item.Size || file.LastWriteTimeUtc != item.ModifiedUtc)
{
item.Sha256 = null;
item.Size = file.Length;
item.ModifiedUtc = file.LastWriteTimeUtc;
}
if (item.Sha256 is not null) return;
var sha = await Task.Run(() => TitanoClient.HashFileAsync(item.Path, ct), ct);
file.Refresh();
if (!file.Exists || file.Length != item.Size || file.LastWriteTimeUtc != item.ModifiedUtc)
{
_queue.Fail(item, UploadFailure.FileChanged, null, UtcNow);
return;
}
item.Sha256 = sha;
_queue.OnChanged();
}
/// <summary>Creates the batch's sequence once, shared by concurrent items of the same batch.</summary>
private async Task<Guid> EnsureSequenceAsync(UploadBatch batch, CancellationToken ct)
{
if (batch.SequenceId is { } known) return known;
if (!_creating.TryGetValue(batch.Id, out var creation))
{
creation = _transport.CreateSequenceAsync(batch.Title, _lifetime.Token);
_creating[batch.Id] = creation;
}
try
{
var id = await creation.WaitAsync(ct);
batch.SequenceId = id;
_queue.OnChanged();
return id;
}
catch when (creation.IsFaulted || creation.IsCanceled)
{
_creating.Remove(batch.Id);
throw;
}
}
/// <summary>
/// BRIEF §0.2 rule 8: deletes only a file the server confirmed by hash, and only when it is still exactly the
/// bytes that were hashed (same size and write time).
/// </summary>
private static void DeleteConfirmedSource(UploadItem item)
{
try
{
var file = new FileInfo(item.Path);
if (!file.Exists) return;
if (file.Length != item.Size || file.LastWriteTimeUtc != item.ModifiedUtc)
{
Log.Warning("upload", $"Kept {item.Path}: it changed after the server verified it");
return;
}
file.Delete();
Log.Info("upload", $"Deleted {item.Path} after the server confirmed SHA-256 {item.Sha256}");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("upload", $"Could not delete {item.Path}", ex);
}
}
private void Requeue(UploadItem item)
{
if (!item.IsActive) return;
item.State = UploadItemState.Queued;
item.BytesSent = 0;
}
private void CancelRunning()
{
foreach (var cts in _running.Values.ToList()) cts.Cancel();
}
private void Refresh()
{
BytesPerSecond = _meter.BytesPerSecond(Now);
var pending = _queue.HasPending;
Status = _queue.IsPaused && pending ? EngineStatus.Paused
: pending && !_transport.IsReady ? EngineStatus.WaitingForServer
: pending || _running.Count > 0 ? EngineStatus.Running
: EngineStatus.Idle;
if (_hadWork && !pending && _running.Count == 0)
{
_hadWork = false;
_meter.Reset();
Drained?.Invoke(this, EventArgs.Empty);
}
}
}
@@ -0,0 +1,41 @@
using System.Collections.Concurrent;
namespace Titano.App.Tests.Uploader;
/// <summary>Runs async test code on one thread, as the UI dispatcher does for the upload engine.</summary>
internal sealed class SingleThreadedContext : SynchronizationContext
{
private readonly BlockingCollection<(SendOrPostCallback Callback, object? State)> _queue = [];
public override void Post(SendOrPostCallback d, object? state)
{
try
{
_queue.Add((d, state));
}
catch (InvalidOperationException)
{
// Late progress reports after the test body finished are dropped.
}
}
public override void Send(SendOrPostCallback d, object? state) => throw new NotSupportedException();
public static void Run(Func<Task> body)
{
var previous = Current;
var context = new SingleThreadedContext();
SetSynchronizationContext(context);
try
{
var task = body();
task.ContinueWith(_ => context._queue.CompleteAdding(), TaskScheduler.Default);
foreach (var (callback, state) in context._queue.GetConsumingEnumerable()) callback(state);
task.GetAwaiter().GetResult();
}
finally
{
SetSynchronizationContext(previous);
}
}
}
@@ -0,0 +1,223 @@
using System.Diagnostics;
using System.IO;
using System.Net;
using System.Security.Cryptography;
using Titano.Protocol;
using Titano.Protocol.Client;
using Titano.Testing;
using Titano.Uploader.Uploads;
namespace Titano.App.Tests.Uploader;
public sealed class UploadEngineTests(TestContext context)
{
private readonly ManualTime _time = new();
[Test]
public void VerifiedUploadsKeepTheSourceByDefault() => SingleThreadedContext.Run(async () =>
{
var queue = Queue(UploadTargetMode.Sessions, 2);
var transport = new FakeTransport();
using var engine = new UploadEngine(queue, transport, () => false, _time);
var confirmed = new List<UploadItem>();
engine.ItemConfirmed += (_, item) => confirmed.Add(item);
engine.Pump();
await Until(() => queue.Items.All(i => i.IsDone));
Assert.True(queue.Items.All(i => i.State == UploadItemState.Verified && i.FrameId is not null));
Assert.True(queue.Items.All(i => File.Exists(i.Path)));
Assert.Equal(Sha256(queue.Items[0].Path), transport.Calls[0].Sha256);
Assert.Equal(2, confirmed.Count);
});
[Test]
public void DeleteAfterUploadRemovesOnlyConfirmedUnchangedFiles() => SingleThreadedContext.Run(async () =>
{
var queue = Queue(UploadTargetMode.Sessions, 3);
var (verified, existing, changed) = (queue.Items[0], queue.Items[1], queue.Items[2]);
var transport = new FakeTransport
{
Respond = item =>
{
if (item == existing) return (UploadOutcome.AlreadyOnServer, null);
// Written to after it was hashed: the bytes on disk are no longer the verified ones.
if (item == changed) File.AppendAllText(item.Path, "more");
return (UploadOutcome.Verified, Guid.CreateVersion7());
},
};
using var engine = new UploadEngine(queue, transport, () => true, _time);
engine.Pump();
await Until(() => queue.Items.All(i => i.IsDone));
Assert.Equal(UploadItemState.Skipped, existing.State);
Assert.False(File.Exists(verified.Path));
Assert.False(File.Exists(existing.Path));
Assert.True(File.Exists(changed.Path), "a file changed after verification is kept");
});
[Test]
public void RejectedFileIsRetriedAfterThirtySeconds() => SingleThreadedContext.Run(async () =>
{
var queue = Queue(UploadTargetMode.Sessions, 1);
var item = queue.Items[0];
var attempts = 0;
var transport = new FakeTransport
{
Respond = _ => ++attempts == 1
? throw new TitanoApiException(HttpStatusCode.UnprocessableEntity, new ApiError("hash_mismatch", "Hash mismatch"))
: (UploadOutcome.Verified, Guid.CreateVersion7()),
};
using var engine = new UploadEngine(queue, transport, () => false, _time);
engine.Pump();
await Until(() => item.State == UploadItemState.Failed);
Assert.Equal(UploadFailure.ServerRejected, item.Failure);
Assert.Equal("Hash mismatch", item.FailureDetail);
Assert.Equal(_time.UtcNow + UploadQueue.RetryDelay, item.NextRetryUtc);
_time.Advance(TimeSpan.FromSeconds(29));
engine.Pump();
Assert.Equal(UploadItemState.Failed, item.State);
_time.Advance(TimeSpan.FromSeconds(1));
engine.Pump();
await Until(() => item.IsDone);
Assert.Equal(1, item.Retries);
Assert.Equal(UploadFailure.None, item.Failure);
});
[Test]
public void UnreachableServerRequeuesWithoutSpendingARetry() => SingleThreadedContext.Run(async () =>
{
var queue = Queue(UploadTargetMode.Sessions, 1);
var item = queue.Items[0];
var transport = new FakeTransport();
transport.Respond = _ =>
{
transport.SetReady(false);
throw new ServerUnavailableException(ServerProblem.Unreachable);
};
using var engine = new UploadEngine(queue, transport, () => false, _time);
engine.Pump();
await Until(() => item.State == UploadItemState.Queued && transport.Calls.Count == 1);
Assert.Equal(0, item.Retries);
Assert.NotNull(item.Sha256);
Assert.Equal(EngineStatus.WaitingForServer, engine.Status);
transport.Respond = _ => (UploadOutcome.Verified, Guid.CreateVersion7());
transport.SetReady(true);
await Until(() => item.IsDone);
});
[Test]
public void NewSequenceIsCreatedOnceForTheBatch() => SingleThreadedContext.Run(async () =>
{
var queue = Queue(UploadTargetMode.NewSequence, 3);
var transport = new FakeTransport();
using var engine = new UploadEngine(queue, transport, () => false, _time);
engine.Pump();
await Until(() => queue.Items.All(i => i.IsDone));
Assert.Equal(1, transport.SequencesCreated);
Assert.True(transport.Calls.All(c => c.SequenceId == FakeTransport.CreatedSequence));
Assert.Equal(FakeTransport.CreatedSequence, queue.Batches.Single().SequenceId);
});
[Test]
public void AtMostTwoFilesAreInFlightAndPauseRequeuesThem() => SingleThreadedContext.Run(async () =>
{
var queue = Queue(UploadTargetMode.Sessions, 4);
var transport = new FakeTransport { Block = true };
using var engine = new UploadEngine(queue, transport, () => false, _time);
engine.Pump();
await Until(() => queue.Items.Count(i => i.State == UploadItemState.Uploading) == UploadEngine.Parallelism);
Assert.Equal(UploadEngine.Parallelism, queue.Items.Count(i => i.IsActive));
engine.Pause();
await Until(() => queue.Items.All(i => i.State == UploadItemState.Queued));
Assert.Equal(EngineStatus.Paused, engine.Status);
transport.Block = false;
engine.Resume();
await Until(() => queue.Items.All(i => i.IsDone));
Assert.False(queue.IsPaused);
});
private UploadQueue Queue(UploadTargetMode mode, int count)
{
var dir = context.CreateTempDirectory();
var queue = new UploadQueue();
var entries = Enumerable.Range(0, count).Select(i =>
{
var path = Path.Combine(dir, $"IMG_{i:0000}.JPG");
File.WriteAllBytes(path, RandomNumberGenerator.GetBytes(4096 + i));
var info = new FileInfo(path);
return new QueueEntry(path, info.Length, info.LastWriteTimeUtc, info.LastWriteTime, "k", null);
}).ToList();
queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = mode, Title = mode == UploadTargetMode.NewSequence ? "Roof" : null }, entries);
return queue;
}
private static string Sha256(string path) => Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(path)));
private static async Task Until(Func<bool> condition)
{
var clock = Stopwatch.StartNew();
while (!condition())
{
if (clock.Elapsed > TimeSpan.FromSeconds(15)) throw new TimeoutException("Condition not reached");
await Task.Delay(5);
}
}
private sealed class ManualTime : TimeProvider
{
private DateTimeOffset _now = new(2026, 9, 28, 18, 0, 0, TimeSpan.Zero);
public DateTime UtcNow => _now.UtcDateTime;
public void Advance(TimeSpan by) => _now += by;
public override DateTimeOffset GetUtcNow() => _now;
}
private sealed class FakeTransport : IUploadTransport
{
public static readonly Guid CreatedSequence = Guid.CreateVersion7();
public event EventHandler? ReadyChanged;
public bool IsReady { get; private set; } = true;
public bool Block { get; set; }
public int SequencesCreated { get; private set; }
public List<(string Path, Guid? SequenceId, string Sha256)> Calls { get; } = [];
public Func<UploadItem, (UploadOutcome, Guid?)> Respond { get; set; } = _ => (UploadOutcome.Verified, Guid.CreateVersion7());
public void SetReady(bool ready)
{
IsReady = ready;
ReadyChanged?.Invoke(this, EventArgs.Empty);
}
public async Task<(UploadOutcome Outcome, Guid? FrameId)> UploadAsync(UploadItem item, Guid? sequenceId, string sha256, IProgress<UploadProgress> progress, CancellationToken ct)
{
Calls.Add((item.Path, sequenceId, sha256));
progress.Report(new UploadProgress(0, item.Size));
while (Block) await Task.Delay(5, ct);
await Task.Yield();
progress.Report(new UploadProgress(item.Size, item.Size));
return Respond(item);
}
public Task<Guid> CreateSequenceAsync(string? title, CancellationToken ct)
{
SequencesCreated++;
return Task.FromResult(CreatedSequence);
}
}
}