diff --git a/src/Titano.Uploader/Sources/PhotoScanner.cs b/src/Titano.Uploader/Sources/PhotoScanner.cs new file mode 100644 index 00000000..85ce2461 --- /dev/null +++ b/src/Titano.Uploader/Sources/PhotoScanner.cs @@ -0,0 +1,82 @@ +using System.IO; +using Titano.Core.Diagnostics; +using Titano.Core.Imaging; + +namespace Titano.Uploader.Sources; + +/// A photo found on disk. is the dropped folder (or the file's folder) it was found under. +public sealed record SourceFile(string Path, long Size, DateTime ModifiedUtc, DateTime CapturedAt, string Root); + +/// Finds supported photos under dropped files, folders and card roots (BRIEF §15: zero analysis). +public static class PhotoScanner +{ + private static readonly EnumerationOptions Recursive = new() + { + RecurseSubdirectories = true, + IgnoreInaccessible = true, + // Skips "System Volume Information", "$RECYCLE.BIN" and camera housekeeping files. + AttributesToSkip = FileAttributes.Hidden | FileAttributes.System, + MatchType = MatchType.Simple, + }; + + /// Supported photos under , in path order, without duplicates. + public static IReadOnlyList Scan(IEnumerable paths, CancellationToken ct = default) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var result = new List(); + foreach (var raw in paths) + { + ct.ThrowIfCancellationRequested(); + string full; + try + { + full = Path.GetFullPath(raw); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException) + { + Log.Warning("scan", $"Ignored path {raw}", ex); + continue; + } + if (Directory.Exists(full)) + { + try + { + foreach (var file in Files(full, ct)) + { + if (seen.Add(file.FullName)) result.Add(ToSource(file, full)); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A card pulled mid-scan: keep what was found; the rest can be dropped again. + Log.Warning("scan", $"Listing of {full} stopped early", ex); + } + } + else if (File.Exists(full) && ImageFormatInfo.IsSupportedExtension(full) && seen.Add(full)) + { + var info = new FileInfo(full); + result.Add(ToSource(info, info.DirectoryName ?? full)); + } + } + return result; + } + + /// Supported photos under , recursively. I/O errors while listing propagate. + public static IEnumerable Files(string folder, CancellationToken ct = default) + { + foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", Recursive)) + { + ct.ThrowIfCancellationRequested(); + if (ImageFormatInfo.IsSupportedExtension(file.Name)) yield return file; + } + } + + public static SourceFile ToSource(FileInfo file, string root) => + new(file.FullName, file.Length, file.LastWriteTimeUtc, CaptureTime(file), root); + + /// + /// Capture time read cheaply: the file's last-write time in local wall-clock, which cameras set to the shot time. + /// Exif dates replace it when the metadata reader lands in Titano.Core. + /// + public static DateTime CaptureTime(FileInfo file) => DateTime.SpecifyKind(file.LastWriteTime, DateTimeKind.Unspecified); +} diff --git a/src/Titano.Uploader/Sources/SessionGrouper.cs b/src/Titano.Uploader/Sources/SessionGrouper.cs new file mode 100644 index 00000000..8102d68c --- /dev/null +++ b/src/Titano.Uploader/Sources/SessionGrouper.cs @@ -0,0 +1,89 @@ +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; + +namespace Titano.Uploader.Sources; + +/// The last session of a source, so later files within the gap join it (watch folders). +public sealed record SessionContinuation(string Key, DateTime LastCapture); + +/// Session key per input frame (input order) and the session the next batch may continue. +public sealed record SessionAssignment(IReadOnlyList Keys, SessionContinuation? Last); + +/// +/// Splits frames of one source into shooting sessions with the card-import rules (BRIEF §13): a gap longer than +/// starts a session; sessions with fewer than frames (test shots) merge +/// into the previous session, or into the next one when they come first. The server turns each key into a sequence. +/// +public static class SessionGrouper +{ + /// SessionGapMinutes default. + public static readonly TimeSpan Gap = TimeSpan.FromMinutes(45); + + /// MinimumSessionFrames default. + public const int MinimumFrames = 8; + + /// Groups ; keys start with so sources never collide. + public static SessionAssignment Assign(IReadOnlyList captureTimes, string sourceTag, SessionContinuation? previous = null) + { + if (captureTimes.Count == 0) return new SessionAssignment([], previous); + var order = Enumerable.Range(0, captureTimes.Count).OrderBy(i => captureTimes[i]).ToArray(); + + // Split on gaps; a group that continues the previous batch keeps its key and is never "small". + var groups = new List(); + var current = new Group { Continues = previous is not null && (captureTimes[order[0]] - previous.LastCapture).Duration() <= Gap }; + for (var n = 0; n < order.Length; n++) + { + var i = order[n]; + if (n > 0 && captureTimes[i] - captureTimes[order[n - 1]] > Gap) + { + groups.Add(current); + current = new Group(); + } + current.Members.Add(i); + } + groups.Add(current); + + var merged = new List(); + foreach (var g in groups) + { + if (merged.Count > 0 && g.Members.Count < MinimumFrames) merged[^1].Members.AddRange(g.Members); + else merged.Add(g); + } + if (merged.Count > 1 && merged[0].Members.Count < MinimumFrames && !merged[0].Continues) + { + merged[0].Members.AddRange(merged[1].Members); + merged.RemoveAt(1); + } + + var keys = new string[captureTimes.Count]; + string? lastKey = null; + foreach (var g in merged) + { + var first = g.Members.Min(i => captureTimes[i]); + lastKey = g.Continues ? previous!.Key : Key(sourceTag, first); + foreach (var i in g.Members) keys[i] = lastKey; + } + var lastCapture = captureTimes[order[^1]]; + if (previous is not null && previous.LastCapture > lastCapture && merged[^1].Continues) lastCapture = previous.LastCapture; + return new SessionAssignment(keys, new SessionContinuation(lastKey!, lastCapture)); + } + + /// Short stable tag for a source folder: 8 hex digits of the SHA-256 of its normalised path. + public static string SourceTag(string root) + { + var normalized = Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)).ToUpperInvariant(); + return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(normalized)))[..8]; + } + + /// {tag}-{yyyyMMdd-HHmmss} of the session's first frame. + public static string Key(string sourceTag, DateTime firstCapture) => + sourceTag + "-" + firstCapture.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture); + + private sealed class Group + { + public List Members { get; } = []; + public bool Continues { get; init; } + } +} diff --git a/src/Titano.Uploader/Text/UploaderText.cs b/src/Titano.Uploader/Text/UploaderText.cs new file mode 100644 index 00000000..bd05ce4f --- /dev/null +++ b/src/Titano.Uploader/Text/UploaderText.cs @@ -0,0 +1,114 @@ +using System.Globalization; +using Titano.Core.Localization; +using Titano.Uploader.Uploads; + +namespace Titano.Uploader.Text; + +/// What the queue is doing, for the end of the queue line. +public enum QueueActivity +{ + Idle, + Running, + Paused, + WaitingForServer, + Done, +} + +/// +/// Culture-aware texts of the uploader: sizes, rates, durations and the queue/progress lines +/// ("4,812 files · 96.2 GB · 38 MB/s · 41 min left"). Every method takes the culture so results are deterministic. +/// +public static class UploaderText +{ + private const string Separator = " · "; + private static readonly string[] ByteUnits = ["B", "KB", "MB", "GB", "TB", "PB"]; + + /// Localized string for in , formatted with . + public static string L(string key, CultureInfo culture, params object?[] args) + { + var format = Loc.Get(key, culture) ?? "⟦" + key + "⟧"; + return args.Length == 0 ? format : string.Format(culture, format, args); + } + + /// Plural-aware lookup of key.one/few/many/other; {0} receives the count. + public static string Plural(string key, long count, CultureInfo culture) + { + var category = PluralRules.Suffix(PluralRules.Select(culture, count)); + var format = Loc.Get(key + "." + category, culture) ?? Loc.Get(key + ".other", culture) ?? "⟦" + key + "⟧"; + return string.Format(culture, format, count); + } + + /// 1,536 → "1.5 KB" (decimal units, as Explorer shows for drives). + public static string Bytes(long bytes, CultureInfo culture) + { + var (value, unit) = Scale(bytes); + var format = unit == 0 || value >= 100 ? "0" : "0.#"; + return value.ToString(format, culture) + " " + ByteUnits[unit]; + } + + /// 38,400,000 B/s → "38 MB/s"; one decimal below 10 units. + public static string Rate(double bytesPerSecond, CultureInfo culture) + { + var (value, unit) = Scale(Math.Max(0, bytesPerSecond)); + var format = unit == 0 || value >= 10 ? "0" : "0.#"; + return L("Uploader.Unit.PerSecond", culture, value.ToString(format, culture) + " " + ByteUnits[unit]); + } + + /// Compact duration: "2 h 05 m", "41 min", "40 s". + public static string Duration(TimeSpan t, CultureInfo culture) + { + if (t.TotalHours >= 24) return L("Common.Duration.Days", culture, (int)t.TotalDays, t.Hours); + if (t.TotalMinutes >= 59.5) return L("Common.Duration.Hours", culture, (int)Math.Round(t.TotalMinutes) / 60, ((int)Math.Round(t.TotalMinutes) % 60).ToString("00", culture)); + if (t.TotalSeconds >= 59.5) return L("Common.Duration.Minutes", culture, (int)Math.Round(t.TotalMinutes)); + return L("Common.Duration.Seconds", culture, (int)Math.Round(t.TotalSeconds)); + } + + /// "4,812 files · 96.2 GB · 38 MB/s · 41 min left", or the files and size followed by the activity. + public static string QueueLine(QueueSummary summary, QueueActivity activity, double bytesPerSecond, TimeSpan? left, CultureInfo culture) + { + if (summary.Files == 0) return L("Uploader.Queue.Empty", culture); + var parts = new List { Plural("Uploader.Queue.Files", summary.Files, culture), Bytes(summary.Bytes, culture) }; + switch (activity) + { + case QueueActivity.Running: + if (bytesPerSecond > 0) parts.Add(Rate(bytesPerSecond, culture)); + if (left is { } l) parts.Add(L("Uploader.Queue.Left", culture, Duration(l, culture))); + break; + case QueueActivity.Paused: + parts.Add(L("Uploader.Queue.Paused", culture)); + break; + case QueueActivity.WaitingForServer: + parts.Add(L("Uploader.Queue.Waiting", culture)); + break; + case QueueActivity.Done: + parts.Add(L("Uploader.Queue.Done", culture)); + break; + } + return string.Join(Separator, parts); + } + + /// "2,214 of 4,812 sent · 2,201 verified by the server · 0 retries", plus failures when there are any. + public static string ProgressLine(QueueSummary summary, CultureInfo culture) + { + var parts = new List + { + L("Uploader.Progress.Sent", culture, summary.Sent, summary.Files), + L("Uploader.Progress.Verified", culture, summary.Verified), + Plural("Uploader.Progress.Retries", summary.Retries, culture), + }; + if (summary.Failed > 0) parts.Add(L("Uploader.Progress.Failed", culture, summary.Failed)); + return string.Join(Separator, parts); + } + + private static (double Value, int Unit) Scale(double bytes) + { + var value = bytes; + var unit = 0; + while (value >= 999.95 && unit < ByteUnits.Length - 1) + { + value /= 1000; + unit++; + } + return (value, unit); + } +} diff --git a/src/Titano.Uploader/Uploads/QueueStore.cs b/src/Titano.Uploader/Uploads/QueueStore.cs new file mode 100644 index 00000000..edf30ac1 --- /dev/null +++ b/src/Titano.Uploader/Uploads/QueueStore.cs @@ -0,0 +1,112 @@ +using System.IO; +using System.Text.Json.Serialization; +using System.Text.Json; +using Titano.Core.Diagnostics; +using Titano.Core.IO; + +namespace Titano.Uploader.Uploads; + +/// +/// uploader-queue.json: the queue survives restarts (BRIEF §15). Server-side upload ids are not stored — +/// restarting an item calls POST /uploads again with the same SHA-256 and the server answers with the chunks +/// it already holds. +/// +public static class QueueStore +{ + public const int Version = 1; + + /// Serializes the queue on the calling (owning) thread; the bytes can be written anywhere. + public static byte[] Serialize(UploadQueue queue) + { + var file = new QueueFile + { + Version = Version, + Paused = queue.IsPaused, + Batches = [.. queue.Batches.Select(b => new BatchRecord(b.Id, b.Mode, b.SequenceId, b.Title))], + Items = + [ + .. queue.Items.Select(i => new ItemRecord( + i.Id, i.BatchId, i.Path, i.Size, i.ModifiedUtc, i.CapturedAt, i.SessionKey, i.WatchRoot, + i.IsActive ? UploadItemState.Queued : i.State, i.Failure, i.FailureDetail, i.Retries, i.Sha256, i.FrameId, i.NextRetryUtc)), + ], + }; + return JsonSerializer.SerializeToUtf8Bytes(file, QueueJson.Default.QueueFile); + } + + /// Loads into ; a missing or corrupt file leaves it empty. + public static void Load(string path, UploadQueue queue) + { + QueueFile? file = null; + try + { + if (File.Exists(path)) file = JsonSerializer.Deserialize(File.ReadAllBytes(path), QueueJson.Default.QueueFile); + } + catch (JsonException ex) + { + Log.Warning("queue", "Queue file unreadable; starting empty", ex); + } + catch (IOException ex) + { + Log.Warning("queue", "Queue file unreadable; starting empty", ex); + } + if (file is null || file.Version != Version) return; + queue.Restore( + file.Batches.Select(b => new UploadBatch { Id = b.Id, Mode = b.Mode, SequenceId = b.SequenceId, Title = b.Title }), + file.Items.Select(r => new UploadItem + { + Id = r.Id, + BatchId = r.BatchId, + Path = r.Path, + Size = r.Size, + ModifiedUtc = r.ModifiedUtc, + CapturedAt = r.CapturedAt, + SessionKey = r.SessionKey, + WatchRoot = r.WatchRoot, + State = r.State, + Failure = r.Failure, + FailureDetail = r.FailureDetail, + Retries = r.Retries, + Sha256 = r.Sha256, + FrameId = r.FrameId, + NextRetryUtc = r.NextRetryUtc, + }), + file.Paused); + } + + /// Commits serialized bytes with temp → fsync → rename. + public static void Write(string path, byte[] content) => AtomicFile.WriteAllBytes(path, content); + + internal sealed class QueueFile + { + public int Version { get; set; } + public bool Paused { get; set; } + public List Batches { get; set; } = []; + public List Items { get; set; } = []; + } + + internal sealed record BatchRecord(Guid Id, UploadTargetMode Mode, Guid? SequenceId, string? Title); + + internal sealed record ItemRecord( + Guid Id, + Guid BatchId, + string Path, + long Size, + DateTime ModifiedUtc, + DateTime? CapturedAt, + string? SessionKey, + string? WatchRoot, + UploadItemState State, + UploadFailure Failure, + string? FailureDetail, + int Retries, + string? Sha256, + Guid? FrameId, + DateTime? NextRetryUtc); +} + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + UseStringEnumConverter = true)] +[JsonSerializable(typeof(QueueStore.QueueFile))] +internal sealed partial class QueueJson : JsonSerializerContext; diff --git a/src/Titano.Uploader/Uploads/QueueSummary.cs b/src/Titano.Uploader/Uploads/QueueSummary.cs new file mode 100644 index 00000000..d361b9b8 --- /dev/null +++ b/src/Titano.Uploader/Uploads/QueueSummary.cs @@ -0,0 +1,115 @@ +namespace Titano.Uploader.Uploads; + +/// Aggregate figures behind the queue line and the progress bar. +public readonly record struct QueueSummary( + int Files, + long Bytes, + int Sent, + int Verified, + int Failed, + int Retries, + long DoneBytes, + string? CurrentFile) +{ + /// 0–1 share of bytes already on the server or in flight. + public double Progress => Bytes <= 0 ? 0 : Math.Clamp(DoneBytes / (double)Bytes, 0, 1); + + /// Bytes still to send, counting only items that can still succeed. + public long RemainingBytes { get; init; } + + public static QueueSummary From(IEnumerable items) + { + int files = 0, sent = 0, verified = 0, failed = 0, retries = 0; + long bytes = 0, done = 0, remaining = 0; + UploadItem? current = null; + foreach (var item in items) + { + files++; + bytes += item.Size; + retries += item.Retries; + switch (item.State) + { + case UploadItemState.Verified or UploadItemState.Skipped: + sent++; + verified++; + done += item.Size; + break; + case UploadItemState.Uploading: + if (item.BytesSent >= item.Size) sent++; + done += Math.Min(item.BytesSent, item.Size); + remaining += Math.Max(0, item.Size - item.BytesSent); + current = item; + break; + case UploadItemState.Hashing: + remaining += item.Size; + current ??= item; + break; + case UploadItemState.Failed: + failed++; + if (item.NextRetryUtc is not null) remaining += item.Size; + break; + default: + remaining += item.Size; + break; + } + } + return new QueueSummary(files, bytes, sent, verified, failed, retries, done, current?.FileName) { RemainingBytes = remaining }; + } +} + +/// Throughput over a sliding window of cumulative byte counts. +public sealed class RateMeter(TimeSpan window) +{ + private readonly List<(TimeSpan At, long Total)> _samples = []; + private long _total; + + public RateMeter() + : this(TimeSpan.FromSeconds(10)) + { + } + + /// Adds sent at (monotonic clock). + public void Add(long bytes, TimeSpan now) + { + // The first sample is the zero baseline so the first chunk counts. + if (_samples.Count == 0) _samples.Add((now, _total)); + _total += bytes; + _samples.Add((now, _total)); + Trim(now); + } + + /// Bytes per second over the window; 0 until the window spans at least one second. + public double BytesPerSecond(TimeSpan now) + { + Trim(now); + if (_samples.Count == 0) return 0; + var (at, total) = _samples[0]; + // Measured up to now, so a stalled transfer decays towards zero. + var span = (now - at).TotalSeconds; + return span < 1 ? 0 : (_total - total) / span; + } + + public void Reset() + { + _samples.Clear(); + _total = 0; + } + + /// Keeps the newest sample at or before now − window as the baseline and drops older ones. + private void Trim(TimeSpan now) + { + var cutoff = now - window; + var drop = 0; + while (drop + 1 < _samples.Count && _samples[drop + 1].At <= cutoff) drop++; + if (drop > 0) _samples.RemoveRange(0, drop); + } + + /// Time to send at ; null without a usable rate. + public static TimeSpan? TimeLeft(long remainingBytes, double bytesPerSecond) + { + if (remainingBytes <= 0) return TimeSpan.Zero; + if (bytesPerSecond < 1024) return null; + var seconds = Math.Ceiling(remainingBytes / bytesPerSecond); + return seconds >= TimeSpan.FromDays(365).TotalSeconds ? null : TimeSpan.FromSeconds(seconds); + } +} diff --git a/src/Titano.Uploader/Uploads/UploadItem.cs b/src/Titano.Uploader/Uploads/UploadItem.cs new file mode 100644 index 00000000..39c7cc12 --- /dev/null +++ b/src/Titano.Uploader/Uploads/UploadItem.cs @@ -0,0 +1,139 @@ +using System.ComponentModel; +using System.Runtime.CompilerServices; + +namespace Titano.Uploader.Uploads; + +/// Per-file state in the upload queue (BRIEF §15). +public enum UploadItemState +{ + Queued, + Hashing, + Uploading, + + /// The server committed the file after checking its SHA-256. + Verified, + + /// The server already held a file with the same SHA-256. + Skipped, + + Failed, +} + +/// Why an item failed; the UI turns it into one localized sentence. +public enum UploadFailure +{ + None, + FileMissing, + FileUnreadable, + + /// The file changed while it was being hashed (still being written). + FileChanged, + + ServerRejected, + Unexpected, +} + +/// Where a batch of files goes on the server. +public enum UploadTargetMode +{ + /// The server groups files by into auto-created sequences. + Sessions, + + /// Every file goes to one existing sequence. + Sequence, + + /// One new sequence is created for the batch with the given title. + NewSequence, +} + +/// Files added together share a target; a new sequence is created once per batch, on first upload. +public sealed class UploadBatch +{ + public required Guid Id { get; init; } + public required UploadTargetMode Mode { get; init; } + + /// Target sequence; for it is filled once the server created it. + public Guid? SequenceId { get; set; } + + public string? Title { get; init; } +} + +/// One file in the queue. Mutated only on the queue's owning (UI) thread. +public sealed class UploadItem : INotifyPropertyChanged +{ + private UploadItemState _state; + private UploadFailure _failure; + private string? _failureDetail; + private int _retries; + private long _bytesSent; + + public required Guid Id { get; init; } + public required Guid BatchId { get; init; } + public required string Path { get; init; } + + /// Size and write time when the file was last hashed or enqueued; a change forces a new hash. + public required long Size { get; set; } + + public required DateTime ModifiedUtc { get; set; } + + /// Camera wall-clock capture time (unspecified kind). + public DateTime? CapturedAt { get; init; } + + public string? SessionKey { get; init; } + + /// Watch folder the file came from; null for files added by hand. + public string? WatchRoot { get; init; } + + public string? Sha256 { get; set; } + public Guid? FrameId { get; set; } + + /// Earliest time a failed item may be retried by the queue. + public DateTime? NextRetryUtc { get; set; } + + public string FileName => System.IO.Path.GetFileName(Path); + + public UploadItemState State + { + get => _state; + set => Set(ref _state, value); + } + + public UploadFailure Failure + { + get => _failure; + set => Set(ref _failure, value); + } + + /// Server message for , otherwise null. + public string? FailureDetail + { + get => _failureDetail; + set => Set(ref _failureDetail, value); + } + + /// Queue-level retries already spent on this item. + public int Retries + { + get => _retries; + set => Set(ref _retries, value); + } + + /// Bytes sent in the current attempt (progress only; not persisted). + public long BytesSent + { + get => _bytesSent; + set => Set(ref _bytesSent, value); + } + + public bool IsDone => _state is UploadItemState.Verified or UploadItemState.Skipped; + public bool IsActive => _state is UploadItemState.Hashing or UploadItemState.Uploading; + + public event PropertyChangedEventHandler? PropertyChanged; + + private void Set(ref T field, T value, [CallerMemberName] string? name = null) + { + if (EqualityComparer.Default.Equals(field, value)) return; + field = value; + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name)); + } +} diff --git a/src/Titano.Uploader/Uploads/UploadQueue.cs b/src/Titano.Uploader/Uploads/UploadQueue.cs new file mode 100644 index 00000000..ffb08ad4 --- /dev/null +++ b/src/Titano.Uploader/Uploads/UploadQueue.cs @@ -0,0 +1,167 @@ +using System.Collections.Concurrent; +using System.Collections.ObjectModel; + +namespace Titano.Uploader.Uploads; + +/// A file about to be enqueued. +public sealed record QueueEntry(string Path, long Size, DateTime ModifiedUtc, DateTime? CapturedAt, string? SessionKey, string? WatchRoot); + +/// +/// Ordered upload queue with its batches. Owned by the UI thread: every mutation happens there, so bindings can +/// observe directly. is the only member safe from other threads. +/// +public sealed class UploadQueue +{ + /// Queue-level retries per item after the client's own transient retries (BRIEF §15). + public const int MaxRetries = 3; + + /// Delay before a failed item is tried again. + public static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(30); + + private readonly Dictionary _batches = []; + private readonly ConcurrentDictionary _paths = new(StringComparer.OrdinalIgnoreCase); + private bool _isPaused; + + public ObservableCollection Items { get; } = []; + + public IReadOnlyCollection Batches => _batches.Values; + + /// True while the user has paused the queue; persisted so a restart keeps it paused. + public bool IsPaused + { + get => _isPaused; + set + { + if (_isPaused == value) return; + _isPaused = value; + OnChanged(); + } + } + + /// Raised after anything that must be persisted changed. + public event EventHandler? Changed; + + /// True while an item for is queued (any state). Thread-safe. + public bool ContainsPath(string path) => _paths.ContainsKey(path); + + /// True while some item still waits for, or is in, an upload attempt (retryable failures included). + public bool HasPending => Items.Any(IsPending); + + /// Appends files not already queued, in the given order, under one batch; returns how many were added. + public int Add(UploadBatch batch, IEnumerable entries) + { + var added = 0; + foreach (var e in entries) + { + if (!_paths.TryAdd(e.Path, 0)) continue; + Items.Add(new UploadItem + { + Id = Guid.CreateVersion7(), + BatchId = batch.Id, + Path = e.Path, + Size = e.Size, + ModifiedUtc = e.ModifiedUtc, + CapturedAt = e.CapturedAt, + SessionKey = e.SessionKey, + WatchRoot = e.WatchRoot, + }); + added++; + } + if (added > 0) + { + _batches[batch.Id] = batch; + OnChanged(); + } + return added; + } + + public UploadBatch? GetBatch(Guid id) => _batches.GetValueOrDefault(id); + + /// + /// The next item to work on: the first queued one, otherwise the first failed one whose retry time has come. + /// + public UploadItem? NextReady(DateTime utcNow) + { + UploadItem? retry = null; + foreach (var item in Items) + { + if (item.State == UploadItemState.Queued) return item; + if (retry is null && IsRetryDue(item, utcNow)) retry = item; + } + return retry; + } + + /// Records a failure and schedules the queue-level retry when retries are left. + public void Fail(UploadItem item, UploadFailure failure, string? detail, DateTime utcNow) + { + item.State = UploadItemState.Failed; + item.Failure = failure; + item.FailureDetail = detail; + item.BytesSent = 0; + item.NextRetryUtc = item.Retries < MaxRetries ? utcNow + RetryDelay : null; + OnChanged(); + } + + /// Puts a failed item back in line now, with a fresh retry budget (user action). + public void Requeue(UploadItem item) + { + if (item.State != UploadItemState.Failed) return; + item.State = UploadItemState.Queued; + item.Failure = UploadFailure.None; + item.FailureDetail = null; + item.Retries = 0; + item.NextRetryUtc = null; + OnChanged(); + } + + /// Drops finished items (verified or skipped) so a new batch starts a fresh count. + public void RemoveCompleted() + { + var removed = false; + for (var i = Items.Count - 1; i >= 0; i--) + { + if (!Items[i].IsDone) continue; + _paths.TryRemove(Items[i].Path, out _); + Items.RemoveAt(i); + removed = true; + } + if (!removed) return; + var used = Items.Select(i => i.BatchId).ToHashSet(); + foreach (var id in _batches.Keys.Where(id => !used.Contains(id)).ToList()) _batches.Remove(id); + OnChanged(); + } + + public void Clear() + { + Items.Clear(); + _paths.Clear(); + _batches.Clear(); + OnChanged(); + } + + /// Signals a persisted-state change made directly on an item. + public void OnChanged() => Changed?.Invoke(this, EventArgs.Empty); + + /// Restores a persisted queue; interrupted attempts go back to . + public void Restore(IEnumerable batches, IEnumerable items, bool paused) + { + Items.Clear(); + _paths.Clear(); + _batches.Clear(); + foreach (var b in batches) _batches[b.Id] = b; + foreach (var item in items) + { + if (!_batches.ContainsKey(item.BatchId) || !_paths.TryAdd(item.Path, 0)) continue; + if (item.IsActive) item.State = UploadItemState.Queued; + Items.Add(item); + } + _isPaused = paused; + } + + private static bool IsPending(UploadItem item) => + item.State is UploadItemState.Queued or UploadItemState.Hashing or UploadItemState.Uploading + || (item.State == UploadItemState.Failed && item.NextRetryUtc is not null); + + private static bool IsRetryDue(UploadItem item, DateTime utcNow) => + item.State == UploadItemState.Failed && item.NextRetryUtc is { } due && due <= utcNow; +} diff --git a/tests/Titano.App.Tests/Uploader/SessionGrouperTests.cs b/tests/Titano.App.Tests/Uploader/SessionGrouperTests.cs new file mode 100644 index 00000000..2335a9d2 --- /dev/null +++ b/tests/Titano.App.Tests/Uploader/SessionGrouperTests.cs @@ -0,0 +1,99 @@ +using System.IO; +using Titano.Testing; +using Titano.Uploader.Sources; + +namespace Titano.App.Tests.Uploader; + +public sealed class SessionGrouperTests +{ + private static readonly DateTime Evening = new(2026, 9, 28, 18, 0, 0); + + [Test] + public void SplitsWhereTheGapExceedsFortyFiveMinutes() + { + var times = Series(Evening, 10).Concat(Series(Evening.AddMinutes(9 + 46), 10)).ToList(); + var result = SessionGrouper.Assign(times, "abcd1234"); + Assert.Equal(2, result.Keys.Distinct().Count()); + Assert.Equal("abcd1234-20260928-180000", result.Keys[0]); + Assert.Equal("abcd1234-20260928-185500", result.Keys[^1]); + } + + [Test] + public void GapOfExactlyFortyFiveMinutesStaysInOneSession() + { + var times = Series(Evening, 10).Concat(Series(Evening.AddMinutes(9 + 45), 10)).ToList(); + Assert.Equal(1, SessionGrouper.Assign(times, "t").Keys.Distinct().Count()); + } + + [Test] + public void TestShotsAfterASessionMergeIntoIt() + { + var times = Series(Evening, 20).Concat(Series(Evening.AddHours(3), 3)).ToList(); + var result = SessionGrouper.Assign(times, "t"); + Assert.Equal(1, result.Keys.Distinct().Count()); + Assert.Equal(Evening.AddHours(3).AddMinutes(2), result.Last!.LastCapture); + } + + [Test] + public void LeadingTestShotsJoinTheFollowingSession() + { + var times = Series(Evening, 3).Concat(Series(Evening.AddHours(2), 30)).ToList(); + var result = SessionGrouper.Assign(times, "t"); + Assert.Equal(1, result.Keys.Distinct().Count()); + Assert.Equal("t-20260928-180000", result.Keys[0]); + } + + [Test] + public void KeysFollowTheInputOrder() + { + var first = Series(Evening, 10); + var second = Series(Evening.AddHours(5), 10); + var shuffled = new List { second[3], first[7], second[0], first[0] }; + shuffled.AddRange(first.Skip(1).Take(6)); + shuffled.AddRange(second.Skip(4)); + var result = SessionGrouper.Assign(shuffled, "t"); + Assert.Equal("t-20260928-230000", result.Keys[0]); + Assert.Equal("t-20260928-180000", result.Keys[1]); + Assert.Equal(result.Keys[0], result.Keys[2]); + Assert.Equal(result.Keys[1], result.Keys[3]); + } + + [Test] + public void NewFramesWithinTheGapContinueThePreviousSession() + { + var previous = new SessionContinuation("t-20260928-180000", Evening.AddMinutes(30)); + var result = SessionGrouper.Assign(Series(Evening.AddMinutes(40), 2), "t", previous); + Assert.SequenceEqual(["t-20260928-180000", "t-20260928-180000"], result.Keys); + Assert.Equal(Evening.AddMinutes(41), result.Last!.LastCapture); + } + + [Test] + public void NewFramesAfterTheGapStartANewSession() + { + var previous = new SessionContinuation("t-20260928-180000", Evening.AddMinutes(30)); + var result = SessionGrouper.Assign(Series(Evening.AddHours(2), 10), "t", previous); + Assert.Equal("t-20260928-200000", result.Keys[0]); + Assert.Equal("t-20260928-200000", result.Last!.Key); + } + + [Test] + public void EmptyInputKeepsThePreviousSession() + { + var previous = new SessionContinuation("k", Evening); + var result = SessionGrouper.Assign([], "t", previous); + Assert.Empty(result.Keys); + Assert.Same(previous, result.Last); + } + + [Test] + public void SourceTagIsStableShortAndPerFolder() + { + var a = SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "roof")); + Assert.Equal(8, a.Length); + Assert.True(a.All(Uri.IsHexDigit)); + Assert.Equal(a, SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "roof") + Path.DirectorySeparatorChar)); + Assert.NotEqual(a, SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "garden"))); + } + + private static List Series(DateTime start, int count) => [.. Enumerable.Range(0, count).Select(i => start.AddMinutes(i))]; +} diff --git a/tests/Titano.App.Tests/Uploader/UploadQueueTests.cs b/tests/Titano.App.Tests/Uploader/UploadQueueTests.cs new file mode 100644 index 00000000..e390ed76 --- /dev/null +++ b/tests/Titano.App.Tests/Uploader/UploadQueueTests.cs @@ -0,0 +1,143 @@ +using System.IO; +using Titano.Testing; +using Titano.Uploader.Uploads; + +namespace Titano.App.Tests.Uploader; + +public sealed class UploadQueueTests(TestContext context) +{ + private static readonly DateTime Now = new(2026, 9, 28, 18, 30, 0, DateTimeKind.Utc); + + [Test] + public void PersistenceRoundTripKeepsBatchesItemsAndStates() + { + var queue = new UploadQueue(); + var sessions = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }; + var created = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.NewSequence, Title = "Harbour sunset", SequenceId = Guid.CreateVersion7() }; + queue.Add(sessions, [Entry("a.cr2", "k1", @"D:\Cam\Roof"), Entry("b.cr2", "k1", null)]); + queue.Add(created, [Entry("c.jpg", null, null)]); + var verified = queue.Items[0]; + verified.State = UploadItemState.Verified; + verified.Sha256 = new string('a', 64); + verified.FrameId = Guid.CreateVersion7(); + var failed = queue.Items[1]; + failed.Retries = 2; + queue.Fail(failed, UploadFailure.ServerRejected, "quota exceeded", Now); + queue.IsPaused = true; + + var path = Path.Combine(context.CreateTempDirectory(), "uploader-queue.json"); + QueueStore.Write(path, QueueStore.Serialize(queue)); + var restored = new UploadQueue(); + QueueStore.Load(path, restored); + + Assert.True(restored.IsPaused); + Assert.Equal(2, restored.Batches.Count); + Assert.Equal("Harbour sunset", restored.GetBatch(created.Id)!.Title); + Assert.Equal(created.SequenceId, restored.GetBatch(created.Id)!.SequenceId); + Assert.Equal(3, restored.Items.Count); + for (var i = 0; i < 3; i++) + { + var (a, b) = (queue.Items[i], restored.Items[i]); + Assert.Equal(a.Id, b.Id); + Assert.Equal(a.BatchId, b.BatchId); + Assert.Equal(a.Path, b.Path); + Assert.Equal(a.Size, b.Size); + Assert.Equal(a.ModifiedUtc, b.ModifiedUtc); + Assert.Equal(a.CapturedAt, b.CapturedAt); + Assert.Equal(a.SessionKey, b.SessionKey); + Assert.Equal(a.WatchRoot, b.WatchRoot); + Assert.Equal(a.State, b.State); + } + Assert.Equal(verified.Sha256, restored.Items[0].Sha256); + Assert.Equal(verified.FrameId, restored.Items[0].FrameId); + Assert.Equal(UploadFailure.ServerRejected, restored.Items[1].Failure); + Assert.Equal("quota exceeded", restored.Items[1].FailureDetail); + Assert.Equal(2, restored.Items[1].Retries); + Assert.Equal(Now + UploadQueue.RetryDelay, restored.Items[1].NextRetryUtc); + Assert.True(restored.ContainsPath("c.jpg")); + } + + [Test] + public void InterruptedTransfersResumeAsQueued() + { + var queue = new UploadQueue(); + queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }, [Entry("a.cr2", null, null), Entry("b.cr2", null, null)]); + queue.Items[0].State = UploadItemState.Uploading; + queue.Items[0].Sha256 = new string('b', 64); + queue.Items[1].State = UploadItemState.Hashing; + + var path = Path.Combine(context.CreateTempDirectory(), "q.json"); + QueueStore.Write(path, QueueStore.Serialize(queue)); + var restored = new UploadQueue(); + QueueStore.Load(path, restored); + + Assert.Equal(UploadItemState.Queued, restored.Items[0].State); + Assert.Equal(UploadItemState.Queued, restored.Items[1].State); + // The known hash is kept: the server resumes from the chunks it already holds. + Assert.Equal(new string('b', 64), restored.Items[0].Sha256); + } + + [Test] + public void CorruptOrMissingFileGivesAnEmptyQueue() + { + var dir = context.CreateTempDirectory(); + var corrupt = Path.Combine(dir, "q.json"); + File.WriteAllText(corrupt, "{ not json"); + var queue = new UploadQueue(); + QueueStore.Load(corrupt, queue); + QueueStore.Load(Path.Combine(dir, "missing.json"), queue); + Assert.Empty(queue.Items); + Assert.False(queue.IsPaused); + } + + [Test] + public void AddingSkipsPathsAlreadyQueued() + { + var queue = new UploadQueue(); + var batch = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }; + Assert.Equal(2, queue.Add(batch, [Entry("a.jpg", null, null), Entry("b.jpg", null, null)])); + Assert.Equal(1, queue.Add(batch, [Entry("b.jpg", null, null), Entry("c.jpg", null, null)])); + Assert.Equal(3, queue.Items.Count); + } + + [Test] + public void FailedItemsComeBackAfterTheDelayUntilRetriesRunOut() + { + var queue = new UploadQueue(); + queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }, [Entry("a.jpg", null, null)]); + var item = queue.Items[0]; + item.State = UploadItemState.Hashing; + queue.Fail(item, UploadFailure.FileUnreadable, null, Now); + Assert.Null(queue.NextReady(Now.AddSeconds(29))); + Assert.Same(item, queue.NextReady(Now + UploadQueue.RetryDelay)); + Assert.True(queue.HasPending); + + item.Retries = UploadQueue.MaxRetries; + queue.Fail(item, UploadFailure.FileUnreadable, null, Now); + Assert.Null(item.NextRetryUtc); + Assert.Null(queue.NextReady(Now.AddDays(1))); + Assert.False(queue.HasPending); + + queue.Requeue(item); + Assert.Equal(0, item.Retries); + Assert.Same(item, queue.NextReady(Now)); + } + + [Test] + public void RemovingCompletedItemsKeepsTheRest() + { + var queue = new UploadQueue(); + var batch = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }; + queue.Add(batch, [Entry("a.jpg", null, null), Entry("b.jpg", null, null), Entry("c.jpg", null, null)]); + queue.Items[0].State = UploadItemState.Verified; + queue.Items[2].State = UploadItemState.Skipped; + queue.RemoveCompleted(); + Assert.Equal(1, queue.Items.Count); + Assert.False(queue.ContainsPath("a.jpg")); + Assert.True(queue.ContainsPath("b.jpg")); + Assert.Equal(1, queue.Batches.Count); + } + + private static QueueEntry Entry(string name, string? session, string? watchRoot) => + new(name, 1_000 + name.Length, new DateTime(2026, 9, 28, 16, 2, 11, DateTimeKind.Utc), new DateTime(2026, 9, 28, 18, 2, 11), session, watchRoot); +} diff --git a/tests/Titano.App.Tests/Uploader/UploaderTextTests.cs b/tests/Titano.App.Tests/Uploader/UploaderTextTests.cs new file mode 100644 index 00000000..d9289fad --- /dev/null +++ b/tests/Titano.App.Tests/Uploader/UploaderTextTests.cs @@ -0,0 +1,124 @@ +using System.Globalization; +using Titano.Testing; +using Titano.Uploader.Text; +using Titano.Uploader.Uploads; + +namespace Titano.App.Tests.Uploader; + +public sealed class UploaderTextTests +{ + private static readonly CultureInfo En = CultureInfo.GetCultureInfo("en"); + private static readonly CultureInfo It = CultureInfo.GetCultureInfo("it"); + + private static readonly QueueSummary Mockup = new(4_812, 96_200_000_000, 2_214, 2_201, 0, 0, 44_000_000_000, "2026-09-28_18-02-11_0413.CR2") + { + RemainingBytes = 93_480_000_000, + }; + + [Test] + public void QueueLineMatchesTheMockup() + { + var left = RateMeter.TimeLeft(Mockup.RemainingBytes, 38_000_000); + Assert.Equal("4,812 files · 96.2 GB · 38 MB/s · 41 min left", UploaderText.QueueLine(Mockup, QueueActivity.Running, 38_000_000, left, En)); + } + + [Test] + public void QueueLineFollowsTheCulture() + { + Assert.Equal("4.812 file · 96,2 GB · 38 MB/s · mancano 41 min", UploaderText.QueueLine(Mockup, QueueActivity.Running, 38_000_000, TimeSpan.FromMinutes(41), It)); + } + + [Test] + public void QueueLineNamesTheActivityWhenNotRunning() + { + Assert.Equal("4,812 files · 96.2 GB · Paused", UploaderText.QueueLine(Mockup, QueueActivity.Paused, 0, null, En)); + Assert.Equal("4,812 files · 96.2 GB · Waiting for server", UploaderText.QueueLine(Mockup, QueueActivity.WaitingForServer, 0, null, En)); + Assert.Equal("1 file · 25 MB · Done", UploaderText.QueueLine(Mockup with { Files = 1, Bytes = 25_000_000 }, QueueActivity.Done, 0, null, En)); + Assert.Equal("Nothing queued", UploaderText.QueueLine(default, QueueActivity.Idle, 0, null, En)); + } + + [Test] + public void RunningLineOmitsRateAndTimeUntilKnown() + { + Assert.Equal("4,812 files · 96.2 GB", UploaderText.QueueLine(Mockup, QueueActivity.Running, 0, null, En)); + } + + [Test] + public void ProgressLineMatchesTheMockup() + { + Assert.Equal("2,214 of 4,812 sent · 2,201 verified by the server · 0 retries", UploaderText.ProgressLine(Mockup, En)); + Assert.Equal("2,214 of 4,812 sent · 2,201 verified by the server · 1 retry · 3 failed", UploaderText.ProgressLine(Mockup with { Retries = 1, Failed = 3 }, En)); + } + + [Theory] + [Case(0.0, "0 B/s")] + [Case(950_000.0, "950 KB/s")] + [Case(4_240_000.0, "4.2 MB/s")] + [Case(38_400_000.0, "38 MB/s")] + [Case(1_260_000_000.0, "1.3 GB/s")] + public void RatesUseDecimalUnits(double bytesPerSecond, string expected) => + Assert.Equal(expected, UploaderText.Rate(bytesPerSecond, En)); + + [Theory] + [Case(40.0, "40 s")] + [Case(299.0, "5 min")] + [Case(3_590.0, "1 h 00 m")] + [Case(7_500.0, "2 h 05 m")] + [Case(93_600.0, "1 d 2 h")] + public void DurationsAreCompact(double seconds, string expected) => + Assert.Equal(expected, UploaderText.Duration(TimeSpan.FromSeconds(seconds), En)); + + [Theory] + [Case(999L, "999 B")] + [Case(1_536L, "1.5 KB")] + [Case(96_200_000_000L, "96.2 GB")] + [Case(1_900_000_000_000L, "1.9 TB")] + public void SizesUseDecimalUnits(long bytes, string expected) => Assert.Equal(expected, UploaderText.Bytes(bytes, En)); + + [Test] + public void TimeLeftNeedsAUsableRate() + { + Assert.Equal(TimeSpan.FromMinutes(41), RateMeter.TimeLeft(93_480_000_000, 38_000_000)); + Assert.Equal(TimeSpan.Zero, RateMeter.TimeLeft(0, 0)); + Assert.Null(RateMeter.TimeLeft(1_000, 10)); + } + + [Test] + public void RateMeterAveragesTheWindowAndDecaysWhenStalled() + { + var meter = new RateMeter(TimeSpan.FromSeconds(10)); + Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.Zero)); + for (var s = 0; s <= 20; s++) meter.Add(4_000_000, TimeSpan.FromSeconds(s)); + Assert.Approximately(4_000_000, meter.BytesPerSecond(TimeSpan.FromSeconds(20)), 1); + // Nothing sent for 20 s: the window holds no transfer any more. + Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.FromSeconds(40))); + meter.Reset(); + Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.FromSeconds(40))); + } + + [Test] + public void SummaryCountsStatesAndBytes() + { + var queue = new UploadQueue(); + queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }, + Enumerable.Range(0, 5).Select(i => new QueueEntry($"{i}.jpg", 100, DateTime.UtcNow, null, null, null))); + queue.Items[0].State = UploadItemState.Verified; + queue.Items[1].State = UploadItemState.Skipped; + queue.Items[2].State = UploadItemState.Uploading; + queue.Items[2].BytesSent = 40; + queue.Items[3].State = UploadItemState.Failed; + queue.Items[3].Retries = 1; + + var s = QueueSummary.From(queue.Items); + Assert.Equal(5, s.Files); + Assert.Equal(500L, s.Bytes); + Assert.Equal(2, s.Sent); + Assert.Equal(2, s.Verified); + Assert.Equal(1, s.Failed); + Assert.Equal(1, s.Retries); + Assert.Equal(240L, s.DoneBytes); + Assert.Equal(160L, s.RemainingBytes); + Assert.Equal("2.jpg", s.CurrentFile); + Assert.Approximately(0.48, s.Progress, 1e-9); + } +}