feat(uploader): persisted upload queue, session grouping and queue texts

Per-file states (queued, hashing, uploading, verified, skipped, failed
with reason), batches with their target (auto sessions, sequence, new
sequence), uploader-queue.json persistence with interrupted transfers
resuming as queued, photo scanning of files/folders/card roots, BRIEF
§13 session keys (45 min gap, test shots merged), throughput meter and
culture-aware queue/progress lines.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 10:27:12 +00:00
1 parent 5e7551c91d
commit 62c7b0d956
10 files changed
+1184

No files matched your search

@@ -0,0 +1,82 @@
using System.IO;
using Titano.Core.Diagnostics;
using Titano.Core.Imaging;
namespace Titano.Uploader.Sources;
/// <summary>A photo found on disk. <see cref="Root"/> is the dropped folder (or the file's folder) it was found under.</summary>
public sealed record SourceFile(string Path, long Size, DateTime ModifiedUtc, DateTime CapturedAt, string Root);
/// <summary>Finds supported photos under dropped files, folders and card roots (BRIEF §15: zero analysis).</summary>
public static class PhotoScanner
{
private static readonly EnumerationOptions Recursive = new()
{
RecurseSubdirectories = true,
IgnoreInaccessible = true,
// Skips "System Volume Information", "$RECYCLE.BIN" and camera housekeeping files.
AttributesToSkip = FileAttributes.Hidden | FileAttributes.System,
MatchType = MatchType.Simple,
};
/// <summary>Supported photos under <paramref name="paths"/>, in path order, without duplicates.</summary>
public static IReadOnlyList<SourceFile> Scan(IEnumerable<string> paths, CancellationToken ct = default)
{
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var result = new List<SourceFile>();
foreach (var raw in paths)
{
ct.ThrowIfCancellationRequested();
string full;
try
{
full = Path.GetFullPath(raw);
}
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
{
Log.Warning("scan", $"Ignored path {raw}", ex);
continue;
}
if (Directory.Exists(full))
{
try
{
foreach (var file in Files(full, ct))
{
if (seen.Add(file.FullName)) result.Add(ToSource(file, full));
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// A card pulled mid-scan: keep what was found; the rest can be dropped again.
Log.Warning("scan", $"Listing of {full} stopped early", ex);
}
}
else if (File.Exists(full) && ImageFormatInfo.IsSupportedExtension(full) && seen.Add(full))
{
var info = new FileInfo(full);
result.Add(ToSource(info, info.DirectoryName ?? full));
}
}
return result;
}
/// <summary>Supported photos under <paramref name="folder"/>, recursively. I/O errors while listing propagate.</summary>
public static IEnumerable<FileInfo> Files(string folder, CancellationToken ct = default)
{
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", Recursive))
{
ct.ThrowIfCancellationRequested();
if (ImageFormatInfo.IsSupportedExtension(file.Name)) yield return file;
}
}
public static SourceFile ToSource(FileInfo file, string root) =>
new(file.FullName, file.Length, file.LastWriteTimeUtc, CaptureTime(file), root);
/// <summary>
/// Capture time read cheaply: the file's last-write time in local wall-clock, which cameras set to the shot time.
/// Exif dates replace it when the metadata reader lands in Titano.Core.
/// </summary>
public static DateTime CaptureTime(FileInfo file) => DateTime.SpecifyKind(file.LastWriteTime, DateTimeKind.Unspecified);
}
@@ -0,0 +1,89 @@
using System.Globalization;
using System.IO;
using System.Security.Cryptography;
using System.Text;
namespace Titano.Uploader.Sources;
/// <summary>The last session of a source, so later files within the gap join it (watch folders).</summary>
public sealed record SessionContinuation(string Key, DateTime LastCapture);
/// <summary>Session key per input frame (input order) and the session the next batch may continue.</summary>
public sealed record SessionAssignment(IReadOnlyList<string> Keys, SessionContinuation? Last);
/// <summary>
/// Splits frames of one source into shooting sessions with the card-import rules (BRIEF §13): a gap longer than
/// <see cref="Gap"/> starts a session; sessions with fewer than <see cref="MinimumFrames"/> frames (test shots) merge
/// into the previous session, or into the next one when they come first. The server turns each key into a sequence.
/// </summary>
public static class SessionGrouper
{
/// <summary><c>SessionGapMinutes</c> default.</summary>
public static readonly TimeSpan Gap = TimeSpan.FromMinutes(45);
/// <summary><c>MinimumSessionFrames</c> default.</summary>
public const int MinimumFrames = 8;
/// <summary>Groups <paramref name="captureTimes"/>; keys start with <paramref name="sourceTag"/> so sources never collide.</summary>
public static SessionAssignment Assign(IReadOnlyList<DateTime> captureTimes, string sourceTag, SessionContinuation? previous = null)
{
if (captureTimes.Count == 0) return new SessionAssignment([], previous);
var order = Enumerable.Range(0, captureTimes.Count).OrderBy(i => captureTimes[i]).ToArray();
// Split on gaps; a group that continues the previous batch keeps its key and is never "small".
var groups = new List<Group>();
var current = new Group { Continues = previous is not null && (captureTimes[order[0]] - previous.LastCapture).Duration() <= Gap };
for (var n = 0; n < order.Length; n++)
{
var i = order[n];
if (n > 0 && captureTimes[i] - captureTimes[order[n - 1]] > Gap)
{
groups.Add(current);
current = new Group();
}
current.Members.Add(i);
}
groups.Add(current);
var merged = new List<Group>();
foreach (var g in groups)
{
if (merged.Count > 0 && g.Members.Count < MinimumFrames) merged[^1].Members.AddRange(g.Members);
else merged.Add(g);
}
if (merged.Count > 1 && merged[0].Members.Count < MinimumFrames && !merged[0].Continues)
{
merged[0].Members.AddRange(merged[1].Members);
merged.RemoveAt(1);
}
var keys = new string[captureTimes.Count];
string? lastKey = null;
foreach (var g in merged)
{
var first = g.Members.Min(i => captureTimes[i]);
lastKey = g.Continues ? previous!.Key : Key(sourceTag, first);
foreach (var i in g.Members) keys[i] = lastKey;
}
var lastCapture = captureTimes[order[^1]];
if (previous is not null && previous.LastCapture > lastCapture && merged[^1].Continues) lastCapture = previous.LastCapture;
return new SessionAssignment(keys, new SessionContinuation(lastKey!, lastCapture));
}
/// <summary>Short stable tag for a source folder: 8 hex digits of the SHA-256 of its normalised path.</summary>
public static string SourceTag(string root)
{
var normalized = Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)).ToUpperInvariant();
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(normalized)))[..8];
}
/// <summary><c>{tag}-{yyyyMMdd-HHmmss}</c> of the session's first frame.</summary>
public static string Key(string sourceTag, DateTime firstCapture) =>
sourceTag + "-" + firstCapture.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
private sealed class Group
{
public List<int> Members { get; } = [];
public bool Continues { get; init; }
}
}
+114
View File
@@ -0,0 +1,114 @@
using System.Globalization;
using Titano.Core.Localization;
using Titano.Uploader.Uploads;
namespace Titano.Uploader.Text;
/// <summary>What the queue is doing, for the end of the queue line.</summary>
public enum QueueActivity
{
Idle,
Running,
Paused,
WaitingForServer,
Done,
}
/// <summary>
/// Culture-aware texts of the uploader: sizes, rates, durations and the queue/progress lines
/// ("4,812 files · 96.2 GB · 38 MB/s · 41 min left"). Every method takes the culture so results are deterministic.
/// </summary>
public static class UploaderText
{
private const string Separator = " · ";
private static readonly string[] ByteUnits = ["B", "KB", "MB", "GB", "TB", "PB"];
/// <summary>Localized string for <paramref name="key"/> in <paramref name="culture"/>, formatted with <paramref name="args"/>.</summary>
public static string L(string key, CultureInfo culture, params object?[] args)
{
var format = Loc.Get(key, culture) ?? "⟦" + key + "⟧";
return args.Length == 0 ? format : string.Format(culture, format, args);
}
/// <summary>Plural-aware lookup of <c>key.one/few/many/other</c>; <c>{0}</c> receives the count.</summary>
public static string Plural(string key, long count, CultureInfo culture)
{
var category = PluralRules.Suffix(PluralRules.Select(culture, count));
var format = Loc.Get(key + "." + category, culture) ?? Loc.Get(key + ".other", culture) ?? "⟦" + key + "⟧";
return string.Format(culture, format, count);
}
/// <summary>1,536 → "1.5 KB" (decimal units, as Explorer shows for drives).</summary>
public static string Bytes(long bytes, CultureInfo culture)
{
var (value, unit) = Scale(bytes);
var format = unit == 0 || value >= 100 ? "0" : "0.#";
return value.ToString(format, culture) + " " + ByteUnits[unit];
}
/// <summary>38,400,000 B/s → "38 MB/s"; one decimal below 10 units.</summary>
public static string Rate(double bytesPerSecond, CultureInfo culture)
{
var (value, unit) = Scale(Math.Max(0, bytesPerSecond));
var format = unit == 0 || value >= 10 ? "0" : "0.#";
return L("Uploader.Unit.PerSecond", culture, value.ToString(format, culture) + " " + ByteUnits[unit]);
}
/// <summary>Compact duration: "2 h 05 m", "41 min", "40 s".</summary>
public static string Duration(TimeSpan t, CultureInfo culture)
{
if (t.TotalHours >= 24) return L("Common.Duration.Days", culture, (int)t.TotalDays, t.Hours);
if (t.TotalMinutes >= 59.5) return L("Common.Duration.Hours", culture, (int)Math.Round(t.TotalMinutes) / 60, ((int)Math.Round(t.TotalMinutes) % 60).ToString("00", culture));
if (t.TotalSeconds >= 59.5) return L("Common.Duration.Minutes", culture, (int)Math.Round(t.TotalMinutes));
return L("Common.Duration.Seconds", culture, (int)Math.Round(t.TotalSeconds));
}
/// <summary>"4,812 files · 96.2 GB · 38 MB/s · 41 min left", or the files and size followed by the activity.</summary>
public static string QueueLine(QueueSummary summary, QueueActivity activity, double bytesPerSecond, TimeSpan? left, CultureInfo culture)
{
if (summary.Files == 0) return L("Uploader.Queue.Empty", culture);
var parts = new List<string> { Plural("Uploader.Queue.Files", summary.Files, culture), Bytes(summary.Bytes, culture) };
switch (activity)
{
case QueueActivity.Running:
if (bytesPerSecond > 0) parts.Add(Rate(bytesPerSecond, culture));
if (left is { } l) parts.Add(L("Uploader.Queue.Left", culture, Duration(l, culture)));
break;
case QueueActivity.Paused:
parts.Add(L("Uploader.Queue.Paused", culture));
break;
case QueueActivity.WaitingForServer:
parts.Add(L("Uploader.Queue.Waiting", culture));
break;
case QueueActivity.Done:
parts.Add(L("Uploader.Queue.Done", culture));
break;
}
return string.Join(Separator, parts);
}
/// <summary>"2,214 of 4,812 sent · 2,201 verified by the server · 0 retries", plus failures when there are any.</summary>
public static string ProgressLine(QueueSummary summary, CultureInfo culture)
{
var parts = new List<string>
{
L("Uploader.Progress.Sent", culture, summary.Sent, summary.Files),
L("Uploader.Progress.Verified", culture, summary.Verified),
Plural("Uploader.Progress.Retries", summary.Retries, culture),
};
if (summary.Failed > 0) parts.Add(L("Uploader.Progress.Failed", culture, summary.Failed));
return string.Join(Separator, parts);
}
private static (double Value, int Unit) Scale(double bytes)
{
var value = bytes;
var unit = 0;
while (value >= 999.95 && unit < ByteUnits.Length - 1)
{
value /= 1000;
unit++;
}
return (value, unit);
}
}
+112
View File
@@ -0,0 +1,112 @@
using System.IO;
using System.Text.Json.Serialization;
using System.Text.Json;
using Titano.Core.Diagnostics;
using Titano.Core.IO;
namespace Titano.Uploader.Uploads;
/// <summary>
/// <c>uploader-queue.json</c>: the queue survives restarts (BRIEF §15). Server-side upload ids are not stored —
/// restarting an item calls <c>POST /uploads</c> again with the same SHA-256 and the server answers with the chunks
/// it already holds.
/// </summary>
public static class QueueStore
{
public const int Version = 1;
/// <summary>Serializes the queue on the calling (owning) thread; the bytes can be written anywhere.</summary>
public static byte[] Serialize(UploadQueue queue)
{
var file = new QueueFile
{
Version = Version,
Paused = queue.IsPaused,
Batches = [.. queue.Batches.Select(b => new BatchRecord(b.Id, b.Mode, b.SequenceId, b.Title))],
Items =
[
.. queue.Items.Select(i => new ItemRecord(
i.Id, i.BatchId, i.Path, i.Size, i.ModifiedUtc, i.CapturedAt, i.SessionKey, i.WatchRoot,
i.IsActive ? UploadItemState.Queued : i.State, i.Failure, i.FailureDetail, i.Retries, i.Sha256, i.FrameId, i.NextRetryUtc)),
],
};
return JsonSerializer.SerializeToUtf8Bytes(file, QueueJson.Default.QueueFile);
}
/// <summary>Loads <paramref name="path"/> into <paramref name="queue"/>; a missing or corrupt file leaves it empty.</summary>
public static void Load(string path, UploadQueue queue)
{
QueueFile? file = null;
try
{
if (File.Exists(path)) file = JsonSerializer.Deserialize(File.ReadAllBytes(path), QueueJson.Default.QueueFile);
}
catch (JsonException ex)
{
Log.Warning("queue", "Queue file unreadable; starting empty", ex);
}
catch (IOException ex)
{
Log.Warning("queue", "Queue file unreadable; starting empty", ex);
}
if (file is null || file.Version != Version) return;
queue.Restore(
file.Batches.Select(b => new UploadBatch { Id = b.Id, Mode = b.Mode, SequenceId = b.SequenceId, Title = b.Title }),
file.Items.Select(r => new UploadItem
{
Id = r.Id,
BatchId = r.BatchId,
Path = r.Path,
Size = r.Size,
ModifiedUtc = r.ModifiedUtc,
CapturedAt = r.CapturedAt,
SessionKey = r.SessionKey,
WatchRoot = r.WatchRoot,
State = r.State,
Failure = r.Failure,
FailureDetail = r.FailureDetail,
Retries = r.Retries,
Sha256 = r.Sha256,
FrameId = r.FrameId,
NextRetryUtc = r.NextRetryUtc,
}),
file.Paused);
}
/// <summary>Commits serialized bytes with temp → fsync → rename.</summary>
public static void Write(string path, byte[] content) => AtomicFile.WriteAllBytes(path, content);
internal sealed class QueueFile
{
public int Version { get; set; }
public bool Paused { get; set; }
public List<BatchRecord> Batches { get; set; } = [];
public List<ItemRecord> Items { get; set; } = [];
}
internal sealed record BatchRecord(Guid Id, UploadTargetMode Mode, Guid? SequenceId, string? Title);
internal sealed record ItemRecord(
Guid Id,
Guid BatchId,
string Path,
long Size,
DateTime ModifiedUtc,
DateTime? CapturedAt,
string? SessionKey,
string? WatchRoot,
UploadItemState State,
UploadFailure Failure,
string? FailureDetail,
int Retries,
string? Sha256,
Guid? FrameId,
DateTime? NextRetryUtc);
}
[JsonSourceGenerationOptions(
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
UseStringEnumConverter = true)]
[JsonSerializable(typeof(QueueStore.QueueFile))]
internal sealed partial class QueueJson : JsonSerializerContext;
+115
View File
@@ -0,0 +1,115 @@
namespace Titano.Uploader.Uploads;
/// <summary>Aggregate figures behind the queue line and the progress bar.</summary>
public readonly record struct QueueSummary(
int Files,
long Bytes,
int Sent,
int Verified,
int Failed,
int Retries,
long DoneBytes,
string? CurrentFile)
{
/// <summary>0–1 share of bytes already on the server or in flight.</summary>
public double Progress => Bytes <= 0 ? 0 : Math.Clamp(DoneBytes / (double)Bytes, 0, 1);
/// <summary>Bytes still to send, counting only items that can still succeed.</summary>
public long RemainingBytes { get; init; }
public static QueueSummary From(IEnumerable<UploadItem> items)
{
int files = 0, sent = 0, verified = 0, failed = 0, retries = 0;
long bytes = 0, done = 0, remaining = 0;
UploadItem? current = null;
foreach (var item in items)
{
files++;
bytes += item.Size;
retries += item.Retries;
switch (item.State)
{
case UploadItemState.Verified or UploadItemState.Skipped:
sent++;
verified++;
done += item.Size;
break;
case UploadItemState.Uploading:
if (item.BytesSent >= item.Size) sent++;
done += Math.Min(item.BytesSent, item.Size);
remaining += Math.Max(0, item.Size - item.BytesSent);
current = item;
break;
case UploadItemState.Hashing:
remaining += item.Size;
current ??= item;
break;
case UploadItemState.Failed:
failed++;
if (item.NextRetryUtc is not null) remaining += item.Size;
break;
default:
remaining += item.Size;
break;
}
}
return new QueueSummary(files, bytes, sent, verified, failed, retries, done, current?.FileName) { RemainingBytes = remaining };
}
}
/// <summary>Throughput over a sliding window of cumulative byte counts.</summary>
public sealed class RateMeter(TimeSpan window)
{
private readonly List<(TimeSpan At, long Total)> _samples = [];
private long _total;
public RateMeter()
: this(TimeSpan.FromSeconds(10))
{
}
/// <summary>Adds <paramref name="bytes"/> sent at <paramref name="now"/> (monotonic clock).</summary>
public void Add(long bytes, TimeSpan now)
{
// The first sample is the zero baseline so the first chunk counts.
if (_samples.Count == 0) _samples.Add((now, _total));
_total += bytes;
_samples.Add((now, _total));
Trim(now);
}
/// <summary>Bytes per second over the window; 0 until the window spans at least one second.</summary>
public double BytesPerSecond(TimeSpan now)
{
Trim(now);
if (_samples.Count == 0) return 0;
var (at, total) = _samples[0];
// Measured up to now, so a stalled transfer decays towards zero.
var span = (now - at).TotalSeconds;
return span < 1 ? 0 : (_total - total) / span;
}
public void Reset()
{
_samples.Clear();
_total = 0;
}
/// <summary>Keeps the newest sample at or before <c>now − window</c> as the baseline and drops older ones.</summary>
private void Trim(TimeSpan now)
{
var cutoff = now - window;
var drop = 0;
while (drop + 1 < _samples.Count && _samples[drop + 1].At <= cutoff) drop++;
if (drop > 0) _samples.RemoveRange(0, drop);
}
/// <summary>Time to send <paramref name="remainingBytes"/> at <paramref name="bytesPerSecond"/>; null without a usable rate.</summary>
public static TimeSpan? TimeLeft(long remainingBytes, double bytesPerSecond)
{
if (remainingBytes <= 0) return TimeSpan.Zero;
if (bytesPerSecond < 1024) return null;
var seconds = Math.Ceiling(remainingBytes / bytesPerSecond);
return seconds >= TimeSpan.FromDays(365).TotalSeconds ? null : TimeSpan.FromSeconds(seconds);
}
}
+139
View File
@@ -0,0 +1,139 @@
using System.ComponentModel;
using System.Runtime.CompilerServices;
namespace Titano.Uploader.Uploads;
/// <summary>Per-file state in the upload queue (BRIEF §15).</summary>
public enum UploadItemState
{
Queued,
Hashing,
Uploading,
/// <summary>The server committed the file after checking its SHA-256.</summary>
Verified,
/// <summary>The server already held a file with the same SHA-256.</summary>
Skipped,
Failed,
}
/// <summary>Why an item failed; the UI turns it into one localized sentence.</summary>
public enum UploadFailure
{
None,
FileMissing,
FileUnreadable,
/// <summary>The file changed while it was being hashed (still being written).</summary>
FileChanged,
ServerRejected,
Unexpected,
}
/// <summary>Where a batch of files goes on the server.</summary>
public enum UploadTargetMode
{
/// <summary>The server groups files by <see cref="UploadItem.SessionKey"/> into auto-created sequences.</summary>
Sessions,
/// <summary>Every file goes to one existing sequence.</summary>
Sequence,
/// <summary>One new sequence is created for the batch with the given title.</summary>
NewSequence,
}
/// <summary>Files added together share a target; a new sequence is created once per batch, on first upload.</summary>
public sealed class UploadBatch
{
public required Guid Id { get; init; }
public required UploadTargetMode Mode { get; init; }
/// <summary>Target sequence; for <see cref="UploadTargetMode.NewSequence"/> it is filled once the server created it.</summary>
public Guid? SequenceId { get; set; }
public string? Title { get; init; }
}
/// <summary>One file in the queue. Mutated only on the queue's owning (UI) thread.</summary>
public sealed class UploadItem : INotifyPropertyChanged
{
private UploadItemState _state;
private UploadFailure _failure;
private string? _failureDetail;
private int _retries;
private long _bytesSent;
public required Guid Id { get; init; }
public required Guid BatchId { get; init; }
public required string Path { get; init; }
/// <summary>Size and write time when the file was last hashed or enqueued; a change forces a new hash.</summary>
public required long Size { get; set; }
public required DateTime ModifiedUtc { get; set; }
/// <summary>Camera wall-clock capture time (unspecified kind).</summary>
public DateTime? CapturedAt { get; init; }
public string? SessionKey { get; init; }
/// <summary>Watch folder the file came from; null for files added by hand.</summary>
public string? WatchRoot { get; init; }
public string? Sha256 { get; set; }
public Guid? FrameId { get; set; }
/// <summary>Earliest time a failed item may be retried by the queue.</summary>
public DateTime? NextRetryUtc { get; set; }
public string FileName => System.IO.Path.GetFileName(Path);
public UploadItemState State
{
get => _state;
set => Set(ref _state, value);
}
public UploadFailure Failure
{
get => _failure;
set => Set(ref _failure, value);
}
/// <summary>Server message for <see cref="UploadFailure.ServerRejected"/>, otherwise null.</summary>
public string? FailureDetail
{
get => _failureDetail;
set => Set(ref _failureDetail, value);
}
/// <summary>Queue-level retries already spent on this item.</summary>
public int Retries
{
get => _retries;
set => Set(ref _retries, value);
}
/// <summary>Bytes sent in the current attempt (progress only; not persisted).</summary>
public long BytesSent
{
get => _bytesSent;
set => Set(ref _bytesSent, value);
}
public bool IsDone => _state is UploadItemState.Verified or UploadItemState.Skipped;
public bool IsActive => _state is UploadItemState.Hashing or UploadItemState.Uploading;
public event PropertyChangedEventHandler? PropertyChanged;
private void Set<T>(ref T field, T value, [CallerMemberName] string? name = null)
{
if (EqualityComparer<T>.Default.Equals(field, value)) return;
field = value;
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name));
}
}
+167
View File
@@ -0,0 +1,167 @@
using System.Collections.Concurrent;
using System.Collections.ObjectModel;
namespace Titano.Uploader.Uploads;
/// <summary>A file about to be enqueued.</summary>
public sealed record QueueEntry(string Path, long Size, DateTime ModifiedUtc, DateTime? CapturedAt, string? SessionKey, string? WatchRoot);
/// <summary>
/// Ordered upload queue with its batches. Owned by the UI thread: every mutation happens there, so bindings can
/// observe <see cref="Items"/> directly. <see cref="ContainsPath"/> is the only member safe from other threads.
/// </summary>
public sealed class UploadQueue
{
/// <summary>Queue-level retries per item after the client's own transient retries (BRIEF §15).</summary>
public const int MaxRetries = 3;
/// <summary>Delay before a failed item is tried again.</summary>
public static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(30);
private readonly Dictionary<Guid, UploadBatch> _batches = [];
private readonly ConcurrentDictionary<string, byte> _paths = new(StringComparer.OrdinalIgnoreCase);
private bool _isPaused;
public ObservableCollection<UploadItem> Items { get; } = [];
public IReadOnlyCollection<UploadBatch> Batches => _batches.Values;
/// <summary>True while the user has paused the queue; persisted so a restart keeps it paused.</summary>
public bool IsPaused
{
get => _isPaused;
set
{
if (_isPaused == value) return;
_isPaused = value;
OnChanged();
}
}
/// <summary>Raised after anything that must be persisted changed.</summary>
public event EventHandler? Changed;
/// <summary>True while an item for <paramref name="path"/> is queued (any state). Thread-safe.</summary>
public bool ContainsPath(string path) => _paths.ContainsKey(path);
/// <summary>True while some item still waits for, or is in, an upload attempt (retryable failures included).</summary>
public bool HasPending => Items.Any(IsPending);
/// <summary>Appends files not already queued, in the given order, under one batch; returns how many were added.</summary>
public int Add(UploadBatch batch, IEnumerable<QueueEntry> entries)
{
var added = 0;
foreach (var e in entries)
{
if (!_paths.TryAdd(e.Path, 0)) continue;
Items.Add(new UploadItem
{
Id = Guid.CreateVersion7(),
BatchId = batch.Id,
Path = e.Path,
Size = e.Size,
ModifiedUtc = e.ModifiedUtc,
CapturedAt = e.CapturedAt,
SessionKey = e.SessionKey,
WatchRoot = e.WatchRoot,
});
added++;
}
if (added > 0)
{
_batches[batch.Id] = batch;
OnChanged();
}
return added;
}
public UploadBatch? GetBatch(Guid id) => _batches.GetValueOrDefault(id);
/// <summary>
/// The next item to work on: the first queued one, otherwise the first failed one whose retry time has come.
/// </summary>
public UploadItem? NextReady(DateTime utcNow)
{
UploadItem? retry = null;
foreach (var item in Items)
{
if (item.State == UploadItemState.Queued) return item;
if (retry is null && IsRetryDue(item, utcNow)) retry = item;
}
return retry;
}
/// <summary>Records a failure and schedules the queue-level retry when retries are left.</summary>
public void Fail(UploadItem item, UploadFailure failure, string? detail, DateTime utcNow)
{
item.State = UploadItemState.Failed;
item.Failure = failure;
item.FailureDetail = detail;
item.BytesSent = 0;
item.NextRetryUtc = item.Retries < MaxRetries ? utcNow + RetryDelay : null;
OnChanged();
}
/// <summary>Puts a failed item back in line now, with a fresh retry budget (user action).</summary>
public void Requeue(UploadItem item)
{
if (item.State != UploadItemState.Failed) return;
item.State = UploadItemState.Queued;
item.Failure = UploadFailure.None;
item.FailureDetail = null;
item.Retries = 0;
item.NextRetryUtc = null;
OnChanged();
}
/// <summary>Drops finished items (verified or skipped) so a new batch starts a fresh count.</summary>
public void RemoveCompleted()
{
var removed = false;
for (var i = Items.Count - 1; i >= 0; i--)
{
if (!Items[i].IsDone) continue;
_paths.TryRemove(Items[i].Path, out _);
Items.RemoveAt(i);
removed = true;
}
if (!removed) return;
var used = Items.Select(i => i.BatchId).ToHashSet();
foreach (var id in _batches.Keys.Where(id => !used.Contains(id)).ToList()) _batches.Remove(id);
OnChanged();
}
public void Clear()
{
Items.Clear();
_paths.Clear();
_batches.Clear();
OnChanged();
}
/// <summary>Signals a persisted-state change made directly on an item.</summary>
public void OnChanged() => Changed?.Invoke(this, EventArgs.Empty);
/// <summary>Restores a persisted queue; interrupted attempts go back to <see cref="UploadItemState.Queued"/>.</summary>
public void Restore(IEnumerable<UploadBatch> batches, IEnumerable<UploadItem> items, bool paused)
{
Items.Clear();
_paths.Clear();
_batches.Clear();
foreach (var b in batches) _batches[b.Id] = b;
foreach (var item in items)
{
if (!_batches.ContainsKey(item.BatchId) || !_paths.TryAdd(item.Path, 0)) continue;
if (item.IsActive) item.State = UploadItemState.Queued;
Items.Add(item);
}
_isPaused = paused;
}
private static bool IsPending(UploadItem item) =>
item.State is UploadItemState.Queued or UploadItemState.Hashing or UploadItemState.Uploading
|| (item.State == UploadItemState.Failed && item.NextRetryUtc is not null);
private static bool IsRetryDue(UploadItem item, DateTime utcNow) =>
item.State == UploadItemState.Failed && item.NextRetryUtc is { } due && due <= utcNow;
}
@@ -0,0 +1,99 @@
using System.IO;
using Titano.Testing;
using Titano.Uploader.Sources;
namespace Titano.App.Tests.Uploader;
public sealed class SessionGrouperTests
{
private static readonly DateTime Evening = new(2026, 9, 28, 18, 0, 0);
[Test]
public void SplitsWhereTheGapExceedsFortyFiveMinutes()
{
var times = Series(Evening, 10).Concat(Series(Evening.AddMinutes(9 + 46), 10)).ToList();
var result = SessionGrouper.Assign(times, "abcd1234");
Assert.Equal(2, result.Keys.Distinct().Count());
Assert.Equal("abcd1234-20260928-180000", result.Keys[0]);
Assert.Equal("abcd1234-20260928-185500", result.Keys[^1]);
}
[Test]
public void GapOfExactlyFortyFiveMinutesStaysInOneSession()
{
var times = Series(Evening, 10).Concat(Series(Evening.AddMinutes(9 + 45), 10)).ToList();
Assert.Equal(1, SessionGrouper.Assign(times, "t").Keys.Distinct().Count());
}
[Test]
public void TestShotsAfterASessionMergeIntoIt()
{
var times = Series(Evening, 20).Concat(Series(Evening.AddHours(3), 3)).ToList();
var result = SessionGrouper.Assign(times, "t");
Assert.Equal(1, result.Keys.Distinct().Count());
Assert.Equal(Evening.AddHours(3).AddMinutes(2), result.Last!.LastCapture);
}
[Test]
public void LeadingTestShotsJoinTheFollowingSession()
{
var times = Series(Evening, 3).Concat(Series(Evening.AddHours(2), 30)).ToList();
var result = SessionGrouper.Assign(times, "t");
Assert.Equal(1, result.Keys.Distinct().Count());
Assert.Equal("t-20260928-180000", result.Keys[0]);
}
[Test]
public void KeysFollowTheInputOrder()
{
var first = Series(Evening, 10);
var second = Series(Evening.AddHours(5), 10);
var shuffled = new List<DateTime> { second[3], first[7], second[0], first[0] };
shuffled.AddRange(first.Skip(1).Take(6));
shuffled.AddRange(second.Skip(4));
var result = SessionGrouper.Assign(shuffled, "t");
Assert.Equal("t-20260928-230000", result.Keys[0]);
Assert.Equal("t-20260928-180000", result.Keys[1]);
Assert.Equal(result.Keys[0], result.Keys[2]);
Assert.Equal(result.Keys[1], result.Keys[3]);
}
[Test]
public void NewFramesWithinTheGapContinueThePreviousSession()
{
var previous = new SessionContinuation("t-20260928-180000", Evening.AddMinutes(30));
var result = SessionGrouper.Assign(Series(Evening.AddMinutes(40), 2), "t", previous);
Assert.SequenceEqual(["t-20260928-180000", "t-20260928-180000"], result.Keys);
Assert.Equal(Evening.AddMinutes(41), result.Last!.LastCapture);
}
[Test]
public void NewFramesAfterTheGapStartANewSession()
{
var previous = new SessionContinuation("t-20260928-180000", Evening.AddMinutes(30));
var result = SessionGrouper.Assign(Series(Evening.AddHours(2), 10), "t", previous);
Assert.Equal("t-20260928-200000", result.Keys[0]);
Assert.Equal("t-20260928-200000", result.Last!.Key);
}
[Test]
public void EmptyInputKeepsThePreviousSession()
{
var previous = new SessionContinuation("k", Evening);
var result = SessionGrouper.Assign([], "t", previous);
Assert.Empty(result.Keys);
Assert.Same(previous, result.Last);
}
[Test]
public void SourceTagIsStableShortAndPerFolder()
{
var a = SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "roof"));
Assert.Equal(8, a.Length);
Assert.True(a.All(Uri.IsHexDigit));
Assert.Equal(a, SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "roof") + Path.DirectorySeparatorChar));
Assert.NotEqual(a, SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "garden")));
}
private static List<DateTime> Series(DateTime start, int count) => [.. Enumerable.Range(0, count).Select(i => start.AddMinutes(i))];
}
@@ -0,0 +1,143 @@
using System.IO;
using Titano.Testing;
using Titano.Uploader.Uploads;
namespace Titano.App.Tests.Uploader;
public sealed class UploadQueueTests(TestContext context)
{
private static readonly DateTime Now = new(2026, 9, 28, 18, 30, 0, DateTimeKind.Utc);
[Test]
public void PersistenceRoundTripKeepsBatchesItemsAndStates()
{
var queue = new UploadQueue();
var sessions = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions };
var created = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.NewSequence, Title = "Harbour sunset", SequenceId = Guid.CreateVersion7() };
queue.Add(sessions, [Entry("a.cr2", "k1", @"D:\Cam\Roof"), Entry("b.cr2", "k1", null)]);
queue.Add(created, [Entry("c.jpg", null, null)]);
var verified = queue.Items[0];
verified.State = UploadItemState.Verified;
verified.Sha256 = new string('a', 64);
verified.FrameId = Guid.CreateVersion7();
var failed = queue.Items[1];
failed.Retries = 2;
queue.Fail(failed, UploadFailure.ServerRejected, "quota exceeded", Now);
queue.IsPaused = true;
var path = Path.Combine(context.CreateTempDirectory(), "uploader-queue.json");
QueueStore.Write(path, QueueStore.Serialize(queue));
var restored = new UploadQueue();
QueueStore.Load(path, restored);
Assert.True(restored.IsPaused);
Assert.Equal(2, restored.Batches.Count);
Assert.Equal("Harbour sunset", restored.GetBatch(created.Id)!.Title);
Assert.Equal(created.SequenceId, restored.GetBatch(created.Id)!.SequenceId);
Assert.Equal(3, restored.Items.Count);
for (var i = 0; i < 3; i++)
{
var (a, b) = (queue.Items[i], restored.Items[i]);
Assert.Equal(a.Id, b.Id);
Assert.Equal(a.BatchId, b.BatchId);
Assert.Equal(a.Path, b.Path);
Assert.Equal(a.Size, b.Size);
Assert.Equal(a.ModifiedUtc, b.ModifiedUtc);
Assert.Equal(a.CapturedAt, b.CapturedAt);
Assert.Equal(a.SessionKey, b.SessionKey);
Assert.Equal(a.WatchRoot, b.WatchRoot);
Assert.Equal(a.State, b.State);
}
Assert.Equal(verified.Sha256, restored.Items[0].Sha256);
Assert.Equal(verified.FrameId, restored.Items[0].FrameId);
Assert.Equal(UploadFailure.ServerRejected, restored.Items[1].Failure);
Assert.Equal("quota exceeded", restored.Items[1].FailureDetail);
Assert.Equal(2, restored.Items[1].Retries);
Assert.Equal(Now + UploadQueue.RetryDelay, restored.Items[1].NextRetryUtc);
Assert.True(restored.ContainsPath("c.jpg"));
}
[Test]
public void InterruptedTransfersResumeAsQueued()
{
var queue = new UploadQueue();
queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }, [Entry("a.cr2", null, null), Entry("b.cr2", null, null)]);
queue.Items[0].State = UploadItemState.Uploading;
queue.Items[0].Sha256 = new string('b', 64);
queue.Items[1].State = UploadItemState.Hashing;
var path = Path.Combine(context.CreateTempDirectory(), "q.json");
QueueStore.Write(path, QueueStore.Serialize(queue));
var restored = new UploadQueue();
QueueStore.Load(path, restored);
Assert.Equal(UploadItemState.Queued, restored.Items[0].State);
Assert.Equal(UploadItemState.Queued, restored.Items[1].State);
// The known hash is kept: the server resumes from the chunks it already holds.
Assert.Equal(new string('b', 64), restored.Items[0].Sha256);
}
[Test]
public void CorruptOrMissingFileGivesAnEmptyQueue()
{
var dir = context.CreateTempDirectory();
var corrupt = Path.Combine(dir, "q.json");
File.WriteAllText(corrupt, "{ not json");
var queue = new UploadQueue();
QueueStore.Load(corrupt, queue);
QueueStore.Load(Path.Combine(dir, "missing.json"), queue);
Assert.Empty(queue.Items);
Assert.False(queue.IsPaused);
}
[Test]
public void AddingSkipsPathsAlreadyQueued()
{
var queue = new UploadQueue();
var batch = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions };
Assert.Equal(2, queue.Add(batch, [Entry("a.jpg", null, null), Entry("b.jpg", null, null)]));
Assert.Equal(1, queue.Add(batch, [Entry("b.jpg", null, null), Entry("c.jpg", null, null)]));
Assert.Equal(3, queue.Items.Count);
}
[Test]
public void FailedItemsComeBackAfterTheDelayUntilRetriesRunOut()
{
var queue = new UploadQueue();
queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }, [Entry("a.jpg", null, null)]);
var item = queue.Items[0];
item.State = UploadItemState.Hashing;
queue.Fail(item, UploadFailure.FileUnreadable, null, Now);
Assert.Null(queue.NextReady(Now.AddSeconds(29)));
Assert.Same(item, queue.NextReady(Now + UploadQueue.RetryDelay));
Assert.True(queue.HasPending);
item.Retries = UploadQueue.MaxRetries;
queue.Fail(item, UploadFailure.FileUnreadable, null, Now);
Assert.Null(item.NextRetryUtc);
Assert.Null(queue.NextReady(Now.AddDays(1)));
Assert.False(queue.HasPending);
queue.Requeue(item);
Assert.Equal(0, item.Retries);
Assert.Same(item, queue.NextReady(Now));
}
[Test]
public void RemovingCompletedItemsKeepsTheRest()
{
var queue = new UploadQueue();
var batch = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions };
queue.Add(batch, [Entry("a.jpg", null, null), Entry("b.jpg", null, null), Entry("c.jpg", null, null)]);
queue.Items[0].State = UploadItemState.Verified;
queue.Items[2].State = UploadItemState.Skipped;
queue.RemoveCompleted();
Assert.Equal(1, queue.Items.Count);
Assert.False(queue.ContainsPath("a.jpg"));
Assert.True(queue.ContainsPath("b.jpg"));
Assert.Equal(1, queue.Batches.Count);
}
private static QueueEntry Entry(string name, string? session, string? watchRoot) =>
new(name, 1_000 + name.Length, new DateTime(2026, 9, 28, 16, 2, 11, DateTimeKind.Utc), new DateTime(2026, 9, 28, 18, 2, 11), session, watchRoot);
}
@@ -0,0 +1,124 @@
using System.Globalization;
using Titano.Testing;
using Titano.Uploader.Text;
using Titano.Uploader.Uploads;
namespace Titano.App.Tests.Uploader;
public sealed class UploaderTextTests
{
private static readonly CultureInfo En = CultureInfo.GetCultureInfo("en");
private static readonly CultureInfo It = CultureInfo.GetCultureInfo("it");
private static readonly QueueSummary Mockup = new(4_812, 96_200_000_000, 2_214, 2_201, 0, 0, 44_000_000_000, "2026-09-28_18-02-11_0413.CR2")
{
RemainingBytes = 93_480_000_000,
};
[Test]
public void QueueLineMatchesTheMockup()
{
var left = RateMeter.TimeLeft(Mockup.RemainingBytes, 38_000_000);
Assert.Equal("4,812 files · 96.2 GB · 38 MB/s · 41 min left", UploaderText.QueueLine(Mockup, QueueActivity.Running, 38_000_000, left, En));
}
[Test]
public void QueueLineFollowsTheCulture()
{
Assert.Equal("4.812 file · 96,2 GB · 38 MB/s · mancano 41 min", UploaderText.QueueLine(Mockup, QueueActivity.Running, 38_000_000, TimeSpan.FromMinutes(41), It));
}
[Test]
public void QueueLineNamesTheActivityWhenNotRunning()
{
Assert.Equal("4,812 files · 96.2 GB · Paused", UploaderText.QueueLine(Mockup, QueueActivity.Paused, 0, null, En));
Assert.Equal("4,812 files · 96.2 GB · Waiting for server", UploaderText.QueueLine(Mockup, QueueActivity.WaitingForServer, 0, null, En));
Assert.Equal("1 file · 25 MB · Done", UploaderText.QueueLine(Mockup with { Files = 1, Bytes = 25_000_000 }, QueueActivity.Done, 0, null, En));
Assert.Equal("Nothing queued", UploaderText.QueueLine(default, QueueActivity.Idle, 0, null, En));
}
[Test]
public void RunningLineOmitsRateAndTimeUntilKnown()
{
Assert.Equal("4,812 files · 96.2 GB", UploaderText.QueueLine(Mockup, QueueActivity.Running, 0, null, En));
}
[Test]
public void ProgressLineMatchesTheMockup()
{
Assert.Equal("2,214 of 4,812 sent · 2,201 verified by the server · 0 retries", UploaderText.ProgressLine(Mockup, En));
Assert.Equal("2,214 of 4,812 sent · 2,201 verified by the server · 1 retry · 3 failed", UploaderText.ProgressLine(Mockup with { Retries = 1, Failed = 3 }, En));
}
[Theory]
[Case(0.0, "0 B/s")]
[Case(950_000.0, "950 KB/s")]
[Case(4_240_000.0, "4.2 MB/s")]
[Case(38_400_000.0, "38 MB/s")]
[Case(1_260_000_000.0, "1.3 GB/s")]
public void RatesUseDecimalUnits(double bytesPerSecond, string expected) =>
Assert.Equal(expected, UploaderText.Rate(bytesPerSecond, En));
[Theory]
[Case(40.0, "40 s")]
[Case(299.0, "5 min")]
[Case(3_590.0, "1 h 00 m")]
[Case(7_500.0, "2 h 05 m")]
[Case(93_600.0, "1 d 2 h")]
public void DurationsAreCompact(double seconds, string expected) =>
Assert.Equal(expected, UploaderText.Duration(TimeSpan.FromSeconds(seconds), En));
[Theory]
[Case(999L, "999 B")]
[Case(1_536L, "1.5 KB")]
[Case(96_200_000_000L, "96.2 GB")]
[Case(1_900_000_000_000L, "1.9 TB")]
public void SizesUseDecimalUnits(long bytes, string expected) => Assert.Equal(expected, UploaderText.Bytes(bytes, En));
[Test]
public void TimeLeftNeedsAUsableRate()
{
Assert.Equal(TimeSpan.FromMinutes(41), RateMeter.TimeLeft(93_480_000_000, 38_000_000));
Assert.Equal(TimeSpan.Zero, RateMeter.TimeLeft(0, 0));
Assert.Null(RateMeter.TimeLeft(1_000, 10));
}
[Test]
public void RateMeterAveragesTheWindowAndDecaysWhenStalled()
{
var meter = new RateMeter(TimeSpan.FromSeconds(10));
Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.Zero));
for (var s = 0; s <= 20; s++) meter.Add(4_000_000, TimeSpan.FromSeconds(s));
Assert.Approximately(4_000_000, meter.BytesPerSecond(TimeSpan.FromSeconds(20)), 1);
// Nothing sent for 20 s: the window holds no transfer any more.
Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.FromSeconds(40)));
meter.Reset();
Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.FromSeconds(40)));
}
[Test]
public void SummaryCountsStatesAndBytes()
{
var queue = new UploadQueue();
queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions },
Enumerable.Range(0, 5).Select(i => new QueueEntry($"{i}.jpg", 100, DateTime.UtcNow, null, null, null)));
queue.Items[0].State = UploadItemState.Verified;
queue.Items[1].State = UploadItemState.Skipped;
queue.Items[2].State = UploadItemState.Uploading;
queue.Items[2].BytesSent = 40;
queue.Items[3].State = UploadItemState.Failed;
queue.Items[3].Retries = 1;
var s = QueueSummary.From(queue.Items);
Assert.Equal(5, s.Files);
Assert.Equal(500L, s.Bytes);
Assert.Equal(2, s.Sent);
Assert.Equal(2, s.Verified);
Assert.Equal(1, s.Failed);
Assert.Equal(1, s.Retries);
Assert.Equal(240L, s.DoneBytes);
Assert.Equal(160L, s.RemainingBytes);
Assert.Equal("2.jpg", s.CurrentFile);
Assert.Approximately(0.48, s.Progress, 1e-9);
}
}