feat(uploader): persisted upload queue, session grouping and queue texts
Per-file states (queued, hashing, uploading, verified, skipped, failed with reason), batches with their target (auto sessions, sequence, new sequence), uploader-queue.json persistence with interrupted transfers resuming as queued, photo scanning of files/folders/card roots, BRIEF §13 session keys (45 min gap, test shots merged), throughput meter and culture-aware queue/progress lines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
10 files changed
+1184
No files matched your search
@@ -0,0 +1,82 @@
|
||||
using System.IO;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Uploader.Sources;
|
||||
|
||||
/// <summary>A photo found on disk. <see cref="Root"/> is the dropped folder (or the file's folder) it was found under.</summary>
|
||||
public sealed record SourceFile(string Path, long Size, DateTime ModifiedUtc, DateTime CapturedAt, string Root);
|
||||
|
||||
/// <summary>Finds supported photos under dropped files, folders and card roots (BRIEF §15: zero analysis).</summary>
|
||||
public static class PhotoScanner
|
||||
{
|
||||
private static readonly EnumerationOptions Recursive = new()
|
||||
{
|
||||
RecurseSubdirectories = true,
|
||||
IgnoreInaccessible = true,
|
||||
// Skips "System Volume Information", "$RECYCLE.BIN" and camera housekeeping files.
|
||||
AttributesToSkip = FileAttributes.Hidden | FileAttributes.System,
|
||||
MatchType = MatchType.Simple,
|
||||
};
|
||||
|
||||
/// <summary>Supported photos under <paramref name="paths"/>, in path order, without duplicates.</summary>
|
||||
public static IReadOnlyList<SourceFile> Scan(IEnumerable<string> paths, CancellationToken ct = default)
|
||||
{
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var result = new List<SourceFile>();
|
||||
foreach (var raw in paths)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
string full;
|
||||
try
|
||||
{
|
||||
full = Path.GetFullPath(raw);
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
Log.Warning("scan", $"Ignored path {raw}", ex);
|
||||
continue;
|
||||
}
|
||||
if (Directory.Exists(full))
|
||||
{
|
||||
try
|
||||
{
|
||||
foreach (var file in Files(full, ct))
|
||||
{
|
||||
if (seen.Add(file.FullName)) result.Add(ToSource(file, full));
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// A card pulled mid-scan: keep what was found; the rest can be dropped again.
|
||||
Log.Warning("scan", $"Listing of {full} stopped early", ex);
|
||||
}
|
||||
}
|
||||
else if (File.Exists(full) && ImageFormatInfo.IsSupportedExtension(full) && seen.Add(full))
|
||||
{
|
||||
var info = new FileInfo(full);
|
||||
result.Add(ToSource(info, info.DirectoryName ?? full));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>Supported photos under <paramref name="folder"/>, recursively. I/O errors while listing propagate.</summary>
|
||||
public static IEnumerable<FileInfo> Files(string folder, CancellationToken ct = default)
|
||||
{
|
||||
foreach (var file in new DirectoryInfo(folder).EnumerateFiles("*", Recursive))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (ImageFormatInfo.IsSupportedExtension(file.Name)) yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
public static SourceFile ToSource(FileInfo file, string root) =>
|
||||
new(file.FullName, file.Length, file.LastWriteTimeUtc, CaptureTime(file), root);
|
||||
|
||||
/// <summary>
|
||||
/// Capture time read cheaply: the file's last-write time in local wall-clock, which cameras set to the shot time.
|
||||
/// Exif dates replace it when the metadata reader lands in Titano.Core.
|
||||
/// </summary>
|
||||
public static DateTime CaptureTime(FileInfo file) => DateTime.SpecifyKind(file.LastWriteTime, DateTimeKind.Unspecified);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace Titano.Uploader.Sources;
|
||||
|
||||
/// <summary>The last session of a source, so later files within the gap join it (watch folders).</summary>
|
||||
public sealed record SessionContinuation(string Key, DateTime LastCapture);
|
||||
|
||||
/// <summary>Session key per input frame (input order) and the session the next batch may continue.</summary>
|
||||
public sealed record SessionAssignment(IReadOnlyList<string> Keys, SessionContinuation? Last);
|
||||
|
||||
/// <summary>
|
||||
/// Splits frames of one source into shooting sessions with the card-import rules (BRIEF §13): a gap longer than
|
||||
/// <see cref="Gap"/> starts a session; sessions with fewer than <see cref="MinimumFrames"/> frames (test shots) merge
|
||||
/// into the previous session, or into the next one when they come first. The server turns each key into a sequence.
|
||||
/// </summary>
|
||||
public static class SessionGrouper
|
||||
{
|
||||
/// <summary><c>SessionGapMinutes</c> default.</summary>
|
||||
public static readonly TimeSpan Gap = TimeSpan.FromMinutes(45);
|
||||
|
||||
/// <summary><c>MinimumSessionFrames</c> default.</summary>
|
||||
public const int MinimumFrames = 8;
|
||||
|
||||
/// <summary>Groups <paramref name="captureTimes"/>; keys start with <paramref name="sourceTag"/> so sources never collide.</summary>
|
||||
public static SessionAssignment Assign(IReadOnlyList<DateTime> captureTimes, string sourceTag, SessionContinuation? previous = null)
|
||||
{
|
||||
if (captureTimes.Count == 0) return new SessionAssignment([], previous);
|
||||
var order = Enumerable.Range(0, captureTimes.Count).OrderBy(i => captureTimes[i]).ToArray();
|
||||
|
||||
// Split on gaps; a group that continues the previous batch keeps its key and is never "small".
|
||||
var groups = new List<Group>();
|
||||
var current = new Group { Continues = previous is not null && (captureTimes[order[0]] - previous.LastCapture).Duration() <= Gap };
|
||||
for (var n = 0; n < order.Length; n++)
|
||||
{
|
||||
var i = order[n];
|
||||
if (n > 0 && captureTimes[i] - captureTimes[order[n - 1]] > Gap)
|
||||
{
|
||||
groups.Add(current);
|
||||
current = new Group();
|
||||
}
|
||||
current.Members.Add(i);
|
||||
}
|
||||
groups.Add(current);
|
||||
|
||||
var merged = new List<Group>();
|
||||
foreach (var g in groups)
|
||||
{
|
||||
if (merged.Count > 0 && g.Members.Count < MinimumFrames) merged[^1].Members.AddRange(g.Members);
|
||||
else merged.Add(g);
|
||||
}
|
||||
if (merged.Count > 1 && merged[0].Members.Count < MinimumFrames && !merged[0].Continues)
|
||||
{
|
||||
merged[0].Members.AddRange(merged[1].Members);
|
||||
merged.RemoveAt(1);
|
||||
}
|
||||
|
||||
var keys = new string[captureTimes.Count];
|
||||
string? lastKey = null;
|
||||
foreach (var g in merged)
|
||||
{
|
||||
var first = g.Members.Min(i => captureTimes[i]);
|
||||
lastKey = g.Continues ? previous!.Key : Key(sourceTag, first);
|
||||
foreach (var i in g.Members) keys[i] = lastKey;
|
||||
}
|
||||
var lastCapture = captureTimes[order[^1]];
|
||||
if (previous is not null && previous.LastCapture > lastCapture && merged[^1].Continues) lastCapture = previous.LastCapture;
|
||||
return new SessionAssignment(keys, new SessionContinuation(lastKey!, lastCapture));
|
||||
}
|
||||
|
||||
/// <summary>Short stable tag for a source folder: 8 hex digits of the SHA-256 of its normalised path.</summary>
|
||||
public static string SourceTag(string root)
|
||||
{
|
||||
var normalized = Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)).ToUpperInvariant();
|
||||
return Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(normalized)))[..8];
|
||||
}
|
||||
|
||||
/// <summary><c>{tag}-{yyyyMMdd-HHmmss}</c> of the session's first frame.</summary>
|
||||
public static string Key(string sourceTag, DateTime firstCapture) =>
|
||||
sourceTag + "-" + firstCapture.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
|
||||
|
||||
private sealed class Group
|
||||
{
|
||||
public List<int> Members { get; } = [];
|
||||
public bool Continues { get; init; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Uploader.Uploads;
|
||||
|
||||
namespace Titano.Uploader.Text;
|
||||
|
||||
/// <summary>What the queue is doing, for the end of the queue line.</summary>
|
||||
public enum QueueActivity
|
||||
{
|
||||
Idle,
|
||||
Running,
|
||||
Paused,
|
||||
WaitingForServer,
|
||||
Done,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Culture-aware texts of the uploader: sizes, rates, durations and the queue/progress lines
|
||||
/// ("4,812 files · 96.2 GB · 38 MB/s · 41 min left"). Every method takes the culture so results are deterministic.
|
||||
/// </summary>
|
||||
public static class UploaderText
|
||||
{
|
||||
private const string Separator = " · ";
|
||||
private static readonly string[] ByteUnits = ["B", "KB", "MB", "GB", "TB", "PB"];
|
||||
|
||||
/// <summary>Localized string for <paramref name="key"/> in <paramref name="culture"/>, formatted with <paramref name="args"/>.</summary>
|
||||
public static string L(string key, CultureInfo culture, params object?[] args)
|
||||
{
|
||||
var format = Loc.Get(key, culture) ?? "⟦" + key + "⟧";
|
||||
return args.Length == 0 ? format : string.Format(culture, format, args);
|
||||
}
|
||||
|
||||
/// <summary>Plural-aware lookup of <c>key.one/few/many/other</c>; <c>{0}</c> receives the count.</summary>
|
||||
public static string Plural(string key, long count, CultureInfo culture)
|
||||
{
|
||||
var category = PluralRules.Suffix(PluralRules.Select(culture, count));
|
||||
var format = Loc.Get(key + "." + category, culture) ?? Loc.Get(key + ".other", culture) ?? "⟦" + key + "⟧";
|
||||
return string.Format(culture, format, count);
|
||||
}
|
||||
|
||||
/// <summary>1,536 → "1.5 KB" (decimal units, as Explorer shows for drives).</summary>
|
||||
public static string Bytes(long bytes, CultureInfo culture)
|
||||
{
|
||||
var (value, unit) = Scale(bytes);
|
||||
var format = unit == 0 || value >= 100 ? "0" : "0.#";
|
||||
return value.ToString(format, culture) + " " + ByteUnits[unit];
|
||||
}
|
||||
|
||||
/// <summary>38,400,000 B/s → "38 MB/s"; one decimal below 10 units.</summary>
|
||||
public static string Rate(double bytesPerSecond, CultureInfo culture)
|
||||
{
|
||||
var (value, unit) = Scale(Math.Max(0, bytesPerSecond));
|
||||
var format = unit == 0 || value >= 10 ? "0" : "0.#";
|
||||
return L("Uploader.Unit.PerSecond", culture, value.ToString(format, culture) + " " + ByteUnits[unit]);
|
||||
}
|
||||
|
||||
/// <summary>Compact duration: "2 h 05 m", "41 min", "40 s".</summary>
|
||||
public static string Duration(TimeSpan t, CultureInfo culture)
|
||||
{
|
||||
if (t.TotalHours >= 24) return L("Common.Duration.Days", culture, (int)t.TotalDays, t.Hours);
|
||||
if (t.TotalMinutes >= 59.5) return L("Common.Duration.Hours", culture, (int)Math.Round(t.TotalMinutes) / 60, ((int)Math.Round(t.TotalMinutes) % 60).ToString("00", culture));
|
||||
if (t.TotalSeconds >= 59.5) return L("Common.Duration.Minutes", culture, (int)Math.Round(t.TotalMinutes));
|
||||
return L("Common.Duration.Seconds", culture, (int)Math.Round(t.TotalSeconds));
|
||||
}
|
||||
|
||||
/// <summary>"4,812 files · 96.2 GB · 38 MB/s · 41 min left", or the files and size followed by the activity.</summary>
|
||||
public static string QueueLine(QueueSummary summary, QueueActivity activity, double bytesPerSecond, TimeSpan? left, CultureInfo culture)
|
||||
{
|
||||
if (summary.Files == 0) return L("Uploader.Queue.Empty", culture);
|
||||
var parts = new List<string> { Plural("Uploader.Queue.Files", summary.Files, culture), Bytes(summary.Bytes, culture) };
|
||||
switch (activity)
|
||||
{
|
||||
case QueueActivity.Running:
|
||||
if (bytesPerSecond > 0) parts.Add(Rate(bytesPerSecond, culture));
|
||||
if (left is { } l) parts.Add(L("Uploader.Queue.Left", culture, Duration(l, culture)));
|
||||
break;
|
||||
case QueueActivity.Paused:
|
||||
parts.Add(L("Uploader.Queue.Paused", culture));
|
||||
break;
|
||||
case QueueActivity.WaitingForServer:
|
||||
parts.Add(L("Uploader.Queue.Waiting", culture));
|
||||
break;
|
||||
case QueueActivity.Done:
|
||||
parts.Add(L("Uploader.Queue.Done", culture));
|
||||
break;
|
||||
}
|
||||
return string.Join(Separator, parts);
|
||||
}
|
||||
|
||||
/// <summary>"2,214 of 4,812 sent · 2,201 verified by the server · 0 retries", plus failures when there are any.</summary>
|
||||
public static string ProgressLine(QueueSummary summary, CultureInfo culture)
|
||||
{
|
||||
var parts = new List<string>
|
||||
{
|
||||
L("Uploader.Progress.Sent", culture, summary.Sent, summary.Files),
|
||||
L("Uploader.Progress.Verified", culture, summary.Verified),
|
||||
Plural("Uploader.Progress.Retries", summary.Retries, culture),
|
||||
};
|
||||
if (summary.Failed > 0) parts.Add(L("Uploader.Progress.Failed", culture, summary.Failed));
|
||||
return string.Join(Separator, parts);
|
||||
}
|
||||
|
||||
private static (double Value, int Unit) Scale(double bytes)
|
||||
{
|
||||
var value = bytes;
|
||||
var unit = 0;
|
||||
while (value >= 999.95 && unit < ByteUnits.Length - 1)
|
||||
{
|
||||
value /= 1000;
|
||||
unit++;
|
||||
}
|
||||
return (value, unit);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
using System.IO;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Uploader.Uploads;
|
||||
|
||||
/// <summary>
|
||||
/// <c>uploader-queue.json</c>: the queue survives restarts (BRIEF §15). Server-side upload ids are not stored —
|
||||
/// restarting an item calls <c>POST /uploads</c> again with the same SHA-256 and the server answers with the chunks
|
||||
/// it already holds.
|
||||
/// </summary>
|
||||
public static class QueueStore
|
||||
{
|
||||
public const int Version = 1;
|
||||
|
||||
/// <summary>Serializes the queue on the calling (owning) thread; the bytes can be written anywhere.</summary>
|
||||
public static byte[] Serialize(UploadQueue queue)
|
||||
{
|
||||
var file = new QueueFile
|
||||
{
|
||||
Version = Version,
|
||||
Paused = queue.IsPaused,
|
||||
Batches = [.. queue.Batches.Select(b => new BatchRecord(b.Id, b.Mode, b.SequenceId, b.Title))],
|
||||
Items =
|
||||
[
|
||||
.. queue.Items.Select(i => new ItemRecord(
|
||||
i.Id, i.BatchId, i.Path, i.Size, i.ModifiedUtc, i.CapturedAt, i.SessionKey, i.WatchRoot,
|
||||
i.IsActive ? UploadItemState.Queued : i.State, i.Failure, i.FailureDetail, i.Retries, i.Sha256, i.FrameId, i.NextRetryUtc)),
|
||||
],
|
||||
};
|
||||
return JsonSerializer.SerializeToUtf8Bytes(file, QueueJson.Default.QueueFile);
|
||||
}
|
||||
|
||||
/// <summary>Loads <paramref name="path"/> into <paramref name="queue"/>; a missing or corrupt file leaves it empty.</summary>
|
||||
public static void Load(string path, UploadQueue queue)
|
||||
{
|
||||
QueueFile? file = null;
|
||||
try
|
||||
{
|
||||
if (File.Exists(path)) file = JsonSerializer.Deserialize(File.ReadAllBytes(path), QueueJson.Default.QueueFile);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
Log.Warning("queue", "Queue file unreadable; starting empty", ex);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
Log.Warning("queue", "Queue file unreadable; starting empty", ex);
|
||||
}
|
||||
if (file is null || file.Version != Version) return;
|
||||
queue.Restore(
|
||||
file.Batches.Select(b => new UploadBatch { Id = b.Id, Mode = b.Mode, SequenceId = b.SequenceId, Title = b.Title }),
|
||||
file.Items.Select(r => new UploadItem
|
||||
{
|
||||
Id = r.Id,
|
||||
BatchId = r.BatchId,
|
||||
Path = r.Path,
|
||||
Size = r.Size,
|
||||
ModifiedUtc = r.ModifiedUtc,
|
||||
CapturedAt = r.CapturedAt,
|
||||
SessionKey = r.SessionKey,
|
||||
WatchRoot = r.WatchRoot,
|
||||
State = r.State,
|
||||
Failure = r.Failure,
|
||||
FailureDetail = r.FailureDetail,
|
||||
Retries = r.Retries,
|
||||
Sha256 = r.Sha256,
|
||||
FrameId = r.FrameId,
|
||||
NextRetryUtc = r.NextRetryUtc,
|
||||
}),
|
||||
file.Paused);
|
||||
}
|
||||
|
||||
/// <summary>Commits serialized bytes with temp → fsync → rename.</summary>
|
||||
public static void Write(string path, byte[] content) => AtomicFile.WriteAllBytes(path, content);
|
||||
|
||||
internal sealed class QueueFile
|
||||
{
|
||||
public int Version { get; set; }
|
||||
public bool Paused { get; set; }
|
||||
public List<BatchRecord> Batches { get; set; } = [];
|
||||
public List<ItemRecord> Items { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed record BatchRecord(Guid Id, UploadTargetMode Mode, Guid? SequenceId, string? Title);
|
||||
|
||||
internal sealed record ItemRecord(
|
||||
Guid Id,
|
||||
Guid BatchId,
|
||||
string Path,
|
||||
long Size,
|
||||
DateTime ModifiedUtc,
|
||||
DateTime? CapturedAt,
|
||||
string? SessionKey,
|
||||
string? WatchRoot,
|
||||
UploadItemState State,
|
||||
UploadFailure Failure,
|
||||
string? FailureDetail,
|
||||
int Retries,
|
||||
string? Sha256,
|
||||
Guid? FrameId,
|
||||
DateTime? NextRetryUtc);
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
UseStringEnumConverter = true)]
|
||||
[JsonSerializable(typeof(QueueStore.QueueFile))]
|
||||
internal sealed partial class QueueJson : JsonSerializerContext;
|
||||
@@ -0,0 +1,115 @@
|
||||
namespace Titano.Uploader.Uploads;
|
||||
|
||||
/// <summary>Aggregate figures behind the queue line and the progress bar.</summary>
|
||||
public readonly record struct QueueSummary(
|
||||
int Files,
|
||||
long Bytes,
|
||||
int Sent,
|
||||
int Verified,
|
||||
int Failed,
|
||||
int Retries,
|
||||
long DoneBytes,
|
||||
string? CurrentFile)
|
||||
{
|
||||
/// <summary>0–1 share of bytes already on the server or in flight.</summary>
|
||||
public double Progress => Bytes <= 0 ? 0 : Math.Clamp(DoneBytes / (double)Bytes, 0, 1);
|
||||
|
||||
/// <summary>Bytes still to send, counting only items that can still succeed.</summary>
|
||||
public long RemainingBytes { get; init; }
|
||||
|
||||
public static QueueSummary From(IEnumerable<UploadItem> items)
|
||||
{
|
||||
int files = 0, sent = 0, verified = 0, failed = 0, retries = 0;
|
||||
long bytes = 0, done = 0, remaining = 0;
|
||||
UploadItem? current = null;
|
||||
foreach (var item in items)
|
||||
{
|
||||
files++;
|
||||
bytes += item.Size;
|
||||
retries += item.Retries;
|
||||
switch (item.State)
|
||||
{
|
||||
case UploadItemState.Verified or UploadItemState.Skipped:
|
||||
sent++;
|
||||
verified++;
|
||||
done += item.Size;
|
||||
break;
|
||||
case UploadItemState.Uploading:
|
||||
if (item.BytesSent >= item.Size) sent++;
|
||||
done += Math.Min(item.BytesSent, item.Size);
|
||||
remaining += Math.Max(0, item.Size - item.BytesSent);
|
||||
current = item;
|
||||
break;
|
||||
case UploadItemState.Hashing:
|
||||
remaining += item.Size;
|
||||
current ??= item;
|
||||
break;
|
||||
case UploadItemState.Failed:
|
||||
failed++;
|
||||
if (item.NextRetryUtc is not null) remaining += item.Size;
|
||||
break;
|
||||
default:
|
||||
remaining += item.Size;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new QueueSummary(files, bytes, sent, verified, failed, retries, done, current?.FileName) { RemainingBytes = remaining };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Throughput over a sliding window of cumulative byte counts.</summary>
|
||||
public sealed class RateMeter(TimeSpan window)
|
||||
{
|
||||
private readonly List<(TimeSpan At, long Total)> _samples = [];
|
||||
private long _total;
|
||||
|
||||
public RateMeter()
|
||||
: this(TimeSpan.FromSeconds(10))
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Adds <paramref name="bytes"/> sent at <paramref name="now"/> (monotonic clock).</summary>
|
||||
public void Add(long bytes, TimeSpan now)
|
||||
{
|
||||
// The first sample is the zero baseline so the first chunk counts.
|
||||
if (_samples.Count == 0) _samples.Add((now, _total));
|
||||
_total += bytes;
|
||||
_samples.Add((now, _total));
|
||||
Trim(now);
|
||||
}
|
||||
|
||||
/// <summary>Bytes per second over the window; 0 until the window spans at least one second.</summary>
|
||||
public double BytesPerSecond(TimeSpan now)
|
||||
{
|
||||
Trim(now);
|
||||
if (_samples.Count == 0) return 0;
|
||||
var (at, total) = _samples[0];
|
||||
// Measured up to now, so a stalled transfer decays towards zero.
|
||||
var span = (now - at).TotalSeconds;
|
||||
return span < 1 ? 0 : (_total - total) / span;
|
||||
}
|
||||
|
||||
public void Reset()
|
||||
{
|
||||
_samples.Clear();
|
||||
_total = 0;
|
||||
}
|
||||
|
||||
/// <summary>Keeps the newest sample at or before <c>now − window</c> as the baseline and drops older ones.</summary>
|
||||
private void Trim(TimeSpan now)
|
||||
{
|
||||
var cutoff = now - window;
|
||||
var drop = 0;
|
||||
while (drop + 1 < _samples.Count && _samples[drop + 1].At <= cutoff) drop++;
|
||||
if (drop > 0) _samples.RemoveRange(0, drop);
|
||||
}
|
||||
|
||||
/// <summary>Time to send <paramref name="remainingBytes"/> at <paramref name="bytesPerSecond"/>; null without a usable rate.</summary>
|
||||
public static TimeSpan? TimeLeft(long remainingBytes, double bytesPerSecond)
|
||||
{
|
||||
if (remainingBytes <= 0) return TimeSpan.Zero;
|
||||
if (bytesPerSecond < 1024) return null;
|
||||
var seconds = Math.Ceiling(remainingBytes / bytesPerSecond);
|
||||
return seconds >= TimeSpan.FromDays(365).TotalSeconds ? null : TimeSpan.FromSeconds(seconds);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
using System.ComponentModel;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace Titano.Uploader.Uploads;
|
||||
|
||||
/// <summary>Per-file state in the upload queue (BRIEF §15).</summary>
|
||||
public enum UploadItemState
|
||||
{
|
||||
Queued,
|
||||
Hashing,
|
||||
Uploading,
|
||||
|
||||
/// <summary>The server committed the file after checking its SHA-256.</summary>
|
||||
Verified,
|
||||
|
||||
/// <summary>The server already held a file with the same SHA-256.</summary>
|
||||
Skipped,
|
||||
|
||||
Failed,
|
||||
}
|
||||
|
||||
/// <summary>Why an item failed; the UI turns it into one localized sentence.</summary>
|
||||
public enum UploadFailure
|
||||
{
|
||||
None,
|
||||
FileMissing,
|
||||
FileUnreadable,
|
||||
|
||||
/// <summary>The file changed while it was being hashed (still being written).</summary>
|
||||
FileChanged,
|
||||
|
||||
ServerRejected,
|
||||
Unexpected,
|
||||
}
|
||||
|
||||
/// <summary>Where a batch of files goes on the server.</summary>
|
||||
public enum UploadTargetMode
|
||||
{
|
||||
/// <summary>The server groups files by <see cref="UploadItem.SessionKey"/> into auto-created sequences.</summary>
|
||||
Sessions,
|
||||
|
||||
/// <summary>Every file goes to one existing sequence.</summary>
|
||||
Sequence,
|
||||
|
||||
/// <summary>One new sequence is created for the batch with the given title.</summary>
|
||||
NewSequence,
|
||||
}
|
||||
|
||||
/// <summary>Files added together share a target; a new sequence is created once per batch, on first upload.</summary>
|
||||
public sealed class UploadBatch
|
||||
{
|
||||
public required Guid Id { get; init; }
|
||||
public required UploadTargetMode Mode { get; init; }
|
||||
|
||||
/// <summary>Target sequence; for <see cref="UploadTargetMode.NewSequence"/> it is filled once the server created it.</summary>
|
||||
public Guid? SequenceId { get; set; }
|
||||
|
||||
public string? Title { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>One file in the queue. Mutated only on the queue's owning (UI) thread.</summary>
|
||||
public sealed class UploadItem : INotifyPropertyChanged
|
||||
{
|
||||
private UploadItemState _state;
|
||||
private UploadFailure _failure;
|
||||
private string? _failureDetail;
|
||||
private int _retries;
|
||||
private long _bytesSent;
|
||||
|
||||
public required Guid Id { get; init; }
|
||||
public required Guid BatchId { get; init; }
|
||||
public required string Path { get; init; }
|
||||
|
||||
/// <summary>Size and write time when the file was last hashed or enqueued; a change forces a new hash.</summary>
|
||||
public required long Size { get; set; }
|
||||
|
||||
public required DateTime ModifiedUtc { get; set; }
|
||||
|
||||
/// <summary>Camera wall-clock capture time (unspecified kind).</summary>
|
||||
public DateTime? CapturedAt { get; init; }
|
||||
|
||||
public string? SessionKey { get; init; }
|
||||
|
||||
/// <summary>Watch folder the file came from; null for files added by hand.</summary>
|
||||
public string? WatchRoot { get; init; }
|
||||
|
||||
public string? Sha256 { get; set; }
|
||||
public Guid? FrameId { get; set; }
|
||||
|
||||
/// <summary>Earliest time a failed item may be retried by the queue.</summary>
|
||||
public DateTime? NextRetryUtc { get; set; }
|
||||
|
||||
public string FileName => System.IO.Path.GetFileName(Path);
|
||||
|
||||
public UploadItemState State
|
||||
{
|
||||
get => _state;
|
||||
set => Set(ref _state, value);
|
||||
}
|
||||
|
||||
public UploadFailure Failure
|
||||
{
|
||||
get => _failure;
|
||||
set => Set(ref _failure, value);
|
||||
}
|
||||
|
||||
/// <summary>Server message for <see cref="UploadFailure.ServerRejected"/>, otherwise null.</summary>
|
||||
public string? FailureDetail
|
||||
{
|
||||
get => _failureDetail;
|
||||
set => Set(ref _failureDetail, value);
|
||||
}
|
||||
|
||||
/// <summary>Queue-level retries already spent on this item.</summary>
|
||||
public int Retries
|
||||
{
|
||||
get => _retries;
|
||||
set => Set(ref _retries, value);
|
||||
}
|
||||
|
||||
/// <summary>Bytes sent in the current attempt (progress only; not persisted).</summary>
|
||||
public long BytesSent
|
||||
{
|
||||
get => _bytesSent;
|
||||
set => Set(ref _bytesSent, value);
|
||||
}
|
||||
|
||||
public bool IsDone => _state is UploadItemState.Verified or UploadItemState.Skipped;
|
||||
public bool IsActive => _state is UploadItemState.Hashing or UploadItemState.Uploading;
|
||||
|
||||
public event PropertyChangedEventHandler? PropertyChanged;
|
||||
|
||||
private void Set<T>(ref T field, T value, [CallerMemberName] string? name = null)
|
||||
{
|
||||
if (EqualityComparer<T>.Default.Equals(field, value)) return;
|
||||
field = value;
|
||||
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(name));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.ObjectModel;
|
||||
|
||||
namespace Titano.Uploader.Uploads;
|
||||
|
||||
/// <summary>A file about to be enqueued.</summary>
|
||||
public sealed record QueueEntry(string Path, long Size, DateTime ModifiedUtc, DateTime? CapturedAt, string? SessionKey, string? WatchRoot);
|
||||
|
||||
/// <summary>
|
||||
/// Ordered upload queue with its batches. Owned by the UI thread: every mutation happens there, so bindings can
|
||||
/// observe <see cref="Items"/> directly. <see cref="ContainsPath"/> is the only member safe from other threads.
|
||||
/// </summary>
|
||||
public sealed class UploadQueue
|
||||
{
|
||||
/// <summary>Queue-level retries per item after the client's own transient retries (BRIEF §15).</summary>
|
||||
public const int MaxRetries = 3;
|
||||
|
||||
/// <summary>Delay before a failed item is tried again.</summary>
|
||||
public static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly Dictionary<Guid, UploadBatch> _batches = [];
|
||||
private readonly ConcurrentDictionary<string, byte> _paths = new(StringComparer.OrdinalIgnoreCase);
|
||||
private bool _isPaused;
|
||||
|
||||
public ObservableCollection<UploadItem> Items { get; } = [];
|
||||
|
||||
public IReadOnlyCollection<UploadBatch> Batches => _batches.Values;
|
||||
|
||||
/// <summary>True while the user has paused the queue; persisted so a restart keeps it paused.</summary>
|
||||
public bool IsPaused
|
||||
{
|
||||
get => _isPaused;
|
||||
set
|
||||
{
|
||||
if (_isPaused == value) return;
|
||||
_isPaused = value;
|
||||
OnChanged();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Raised after anything that must be persisted changed.</summary>
|
||||
public event EventHandler? Changed;
|
||||
|
||||
/// <summary>True while an item for <paramref name="path"/> is queued (any state). Thread-safe.</summary>
|
||||
public bool ContainsPath(string path) => _paths.ContainsKey(path);
|
||||
|
||||
/// <summary>True while some item still waits for, or is in, an upload attempt (retryable failures included).</summary>
|
||||
public bool HasPending => Items.Any(IsPending);
|
||||
|
||||
/// <summary>Appends files not already queued, in the given order, under one batch; returns how many were added.</summary>
|
||||
public int Add(UploadBatch batch, IEnumerable<QueueEntry> entries)
|
||||
{
|
||||
var added = 0;
|
||||
foreach (var e in entries)
|
||||
{
|
||||
if (!_paths.TryAdd(e.Path, 0)) continue;
|
||||
Items.Add(new UploadItem
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
BatchId = batch.Id,
|
||||
Path = e.Path,
|
||||
Size = e.Size,
|
||||
ModifiedUtc = e.ModifiedUtc,
|
||||
CapturedAt = e.CapturedAt,
|
||||
SessionKey = e.SessionKey,
|
||||
WatchRoot = e.WatchRoot,
|
||||
});
|
||||
added++;
|
||||
}
|
||||
if (added > 0)
|
||||
{
|
||||
_batches[batch.Id] = batch;
|
||||
OnChanged();
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
public UploadBatch? GetBatch(Guid id) => _batches.GetValueOrDefault(id);
|
||||
|
||||
/// <summary>
|
||||
/// The next item to work on: the first queued one, otherwise the first failed one whose retry time has come.
|
||||
/// </summary>
|
||||
public UploadItem? NextReady(DateTime utcNow)
|
||||
{
|
||||
UploadItem? retry = null;
|
||||
foreach (var item in Items)
|
||||
{
|
||||
if (item.State == UploadItemState.Queued) return item;
|
||||
if (retry is null && IsRetryDue(item, utcNow)) retry = item;
|
||||
}
|
||||
return retry;
|
||||
}
|
||||
|
||||
/// <summary>Records a failure and schedules the queue-level retry when retries are left.</summary>
|
||||
public void Fail(UploadItem item, UploadFailure failure, string? detail, DateTime utcNow)
|
||||
{
|
||||
item.State = UploadItemState.Failed;
|
||||
item.Failure = failure;
|
||||
item.FailureDetail = detail;
|
||||
item.BytesSent = 0;
|
||||
item.NextRetryUtc = item.Retries < MaxRetries ? utcNow + RetryDelay : null;
|
||||
OnChanged();
|
||||
}
|
||||
|
||||
/// <summary>Puts a failed item back in line now, with a fresh retry budget (user action).</summary>
|
||||
public void Requeue(UploadItem item)
|
||||
{
|
||||
if (item.State != UploadItemState.Failed) return;
|
||||
item.State = UploadItemState.Queued;
|
||||
item.Failure = UploadFailure.None;
|
||||
item.FailureDetail = null;
|
||||
item.Retries = 0;
|
||||
item.NextRetryUtc = null;
|
||||
OnChanged();
|
||||
}
|
||||
|
||||
/// <summary>Drops finished items (verified or skipped) so a new batch starts a fresh count.</summary>
|
||||
public void RemoveCompleted()
|
||||
{
|
||||
var removed = false;
|
||||
for (var i = Items.Count - 1; i >= 0; i--)
|
||||
{
|
||||
if (!Items[i].IsDone) continue;
|
||||
_paths.TryRemove(Items[i].Path, out _);
|
||||
Items.RemoveAt(i);
|
||||
removed = true;
|
||||
}
|
||||
if (!removed) return;
|
||||
var used = Items.Select(i => i.BatchId).ToHashSet();
|
||||
foreach (var id in _batches.Keys.Where(id => !used.Contains(id)).ToList()) _batches.Remove(id);
|
||||
OnChanged();
|
||||
}
|
||||
|
||||
public void Clear()
|
||||
{
|
||||
Items.Clear();
|
||||
_paths.Clear();
|
||||
_batches.Clear();
|
||||
OnChanged();
|
||||
}
|
||||
|
||||
/// <summary>Signals a persisted-state change made directly on an item.</summary>
|
||||
public void OnChanged() => Changed?.Invoke(this, EventArgs.Empty);
|
||||
|
||||
/// <summary>Restores a persisted queue; interrupted attempts go back to <see cref="UploadItemState.Queued"/>.</summary>
|
||||
public void Restore(IEnumerable<UploadBatch> batches, IEnumerable<UploadItem> items, bool paused)
|
||||
{
|
||||
Items.Clear();
|
||||
_paths.Clear();
|
||||
_batches.Clear();
|
||||
foreach (var b in batches) _batches[b.Id] = b;
|
||||
foreach (var item in items)
|
||||
{
|
||||
if (!_batches.ContainsKey(item.BatchId) || !_paths.TryAdd(item.Path, 0)) continue;
|
||||
if (item.IsActive) item.State = UploadItemState.Queued;
|
||||
Items.Add(item);
|
||||
}
|
||||
_isPaused = paused;
|
||||
}
|
||||
|
||||
private static bool IsPending(UploadItem item) =>
|
||||
item.State is UploadItemState.Queued or UploadItemState.Hashing or UploadItemState.Uploading
|
||||
|| (item.State == UploadItemState.Failed && item.NextRetryUtc is not null);
|
||||
|
||||
private static bool IsRetryDue(UploadItem item, DateTime utcNow) =>
|
||||
item.State == UploadItemState.Failed && item.NextRetryUtc is { } due && due <= utcNow;
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
using System.IO;
|
||||
using Titano.Testing;
|
||||
using Titano.Uploader.Sources;
|
||||
|
||||
namespace Titano.App.Tests.Uploader;
|
||||
|
||||
public sealed class SessionGrouperTests
|
||||
{
|
||||
private static readonly DateTime Evening = new(2026, 9, 28, 18, 0, 0);
|
||||
|
||||
[Test]
|
||||
public void SplitsWhereTheGapExceedsFortyFiveMinutes()
|
||||
{
|
||||
var times = Series(Evening, 10).Concat(Series(Evening.AddMinutes(9 + 46), 10)).ToList();
|
||||
var result = SessionGrouper.Assign(times, "abcd1234");
|
||||
Assert.Equal(2, result.Keys.Distinct().Count());
|
||||
Assert.Equal("abcd1234-20260928-180000", result.Keys[0]);
|
||||
Assert.Equal("abcd1234-20260928-185500", result.Keys[^1]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GapOfExactlyFortyFiveMinutesStaysInOneSession()
|
||||
{
|
||||
var times = Series(Evening, 10).Concat(Series(Evening.AddMinutes(9 + 45), 10)).ToList();
|
||||
Assert.Equal(1, SessionGrouper.Assign(times, "t").Keys.Distinct().Count());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TestShotsAfterASessionMergeIntoIt()
|
||||
{
|
||||
var times = Series(Evening, 20).Concat(Series(Evening.AddHours(3), 3)).ToList();
|
||||
var result = SessionGrouper.Assign(times, "t");
|
||||
Assert.Equal(1, result.Keys.Distinct().Count());
|
||||
Assert.Equal(Evening.AddHours(3).AddMinutes(2), result.Last!.LastCapture);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void LeadingTestShotsJoinTheFollowingSession()
|
||||
{
|
||||
var times = Series(Evening, 3).Concat(Series(Evening.AddHours(2), 30)).ToList();
|
||||
var result = SessionGrouper.Assign(times, "t");
|
||||
Assert.Equal(1, result.Keys.Distinct().Count());
|
||||
Assert.Equal("t-20260928-180000", result.Keys[0]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void KeysFollowTheInputOrder()
|
||||
{
|
||||
var first = Series(Evening, 10);
|
||||
var second = Series(Evening.AddHours(5), 10);
|
||||
var shuffled = new List<DateTime> { second[3], first[7], second[0], first[0] };
|
||||
shuffled.AddRange(first.Skip(1).Take(6));
|
||||
shuffled.AddRange(second.Skip(4));
|
||||
var result = SessionGrouper.Assign(shuffled, "t");
|
||||
Assert.Equal("t-20260928-230000", result.Keys[0]);
|
||||
Assert.Equal("t-20260928-180000", result.Keys[1]);
|
||||
Assert.Equal(result.Keys[0], result.Keys[2]);
|
||||
Assert.Equal(result.Keys[1], result.Keys[3]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NewFramesWithinTheGapContinueThePreviousSession()
|
||||
{
|
||||
var previous = new SessionContinuation("t-20260928-180000", Evening.AddMinutes(30));
|
||||
var result = SessionGrouper.Assign(Series(Evening.AddMinutes(40), 2), "t", previous);
|
||||
Assert.SequenceEqual(["t-20260928-180000", "t-20260928-180000"], result.Keys);
|
||||
Assert.Equal(Evening.AddMinutes(41), result.Last!.LastCapture);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NewFramesAfterTheGapStartANewSession()
|
||||
{
|
||||
var previous = new SessionContinuation("t-20260928-180000", Evening.AddMinutes(30));
|
||||
var result = SessionGrouper.Assign(Series(Evening.AddHours(2), 10), "t", previous);
|
||||
Assert.Equal("t-20260928-200000", result.Keys[0]);
|
||||
Assert.Equal("t-20260928-200000", result.Last!.Key);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EmptyInputKeepsThePreviousSession()
|
||||
{
|
||||
var previous = new SessionContinuation("k", Evening);
|
||||
var result = SessionGrouper.Assign([], "t", previous);
|
||||
Assert.Empty(result.Keys);
|
||||
Assert.Same(previous, result.Last);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SourceTagIsStableShortAndPerFolder()
|
||||
{
|
||||
var a = SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "roof"));
|
||||
Assert.Equal(8, a.Length);
|
||||
Assert.True(a.All(Uri.IsHexDigit));
|
||||
Assert.Equal(a, SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "roof") + Path.DirectorySeparatorChar));
|
||||
Assert.NotEqual(a, SessionGrouper.SourceTag(Path.Combine(Path.GetTempPath(), "cam", "garden")));
|
||||
}
|
||||
|
||||
private static List<DateTime> Series(DateTime start, int count) => [.. Enumerable.Range(0, count).Select(i => start.AddMinutes(i))];
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
using System.IO;
|
||||
using Titano.Testing;
|
||||
using Titano.Uploader.Uploads;
|
||||
|
||||
namespace Titano.App.Tests.Uploader;
|
||||
|
||||
public sealed class UploadQueueTests(TestContext context)
|
||||
{
|
||||
private static readonly DateTime Now = new(2026, 9, 28, 18, 30, 0, DateTimeKind.Utc);
|
||||
|
||||
[Test]
|
||||
public void PersistenceRoundTripKeepsBatchesItemsAndStates()
|
||||
{
|
||||
var queue = new UploadQueue();
|
||||
var sessions = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions };
|
||||
var created = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.NewSequence, Title = "Harbour sunset", SequenceId = Guid.CreateVersion7() };
|
||||
queue.Add(sessions, [Entry("a.cr2", "k1", @"D:\Cam\Roof"), Entry("b.cr2", "k1", null)]);
|
||||
queue.Add(created, [Entry("c.jpg", null, null)]);
|
||||
var verified = queue.Items[0];
|
||||
verified.State = UploadItemState.Verified;
|
||||
verified.Sha256 = new string('a', 64);
|
||||
verified.FrameId = Guid.CreateVersion7();
|
||||
var failed = queue.Items[1];
|
||||
failed.Retries = 2;
|
||||
queue.Fail(failed, UploadFailure.ServerRejected, "quota exceeded", Now);
|
||||
queue.IsPaused = true;
|
||||
|
||||
var path = Path.Combine(context.CreateTempDirectory(), "uploader-queue.json");
|
||||
QueueStore.Write(path, QueueStore.Serialize(queue));
|
||||
var restored = new UploadQueue();
|
||||
QueueStore.Load(path, restored);
|
||||
|
||||
Assert.True(restored.IsPaused);
|
||||
Assert.Equal(2, restored.Batches.Count);
|
||||
Assert.Equal("Harbour sunset", restored.GetBatch(created.Id)!.Title);
|
||||
Assert.Equal(created.SequenceId, restored.GetBatch(created.Id)!.SequenceId);
|
||||
Assert.Equal(3, restored.Items.Count);
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
var (a, b) = (queue.Items[i], restored.Items[i]);
|
||||
Assert.Equal(a.Id, b.Id);
|
||||
Assert.Equal(a.BatchId, b.BatchId);
|
||||
Assert.Equal(a.Path, b.Path);
|
||||
Assert.Equal(a.Size, b.Size);
|
||||
Assert.Equal(a.ModifiedUtc, b.ModifiedUtc);
|
||||
Assert.Equal(a.CapturedAt, b.CapturedAt);
|
||||
Assert.Equal(a.SessionKey, b.SessionKey);
|
||||
Assert.Equal(a.WatchRoot, b.WatchRoot);
|
||||
Assert.Equal(a.State, b.State);
|
||||
}
|
||||
Assert.Equal(verified.Sha256, restored.Items[0].Sha256);
|
||||
Assert.Equal(verified.FrameId, restored.Items[0].FrameId);
|
||||
Assert.Equal(UploadFailure.ServerRejected, restored.Items[1].Failure);
|
||||
Assert.Equal("quota exceeded", restored.Items[1].FailureDetail);
|
||||
Assert.Equal(2, restored.Items[1].Retries);
|
||||
Assert.Equal(Now + UploadQueue.RetryDelay, restored.Items[1].NextRetryUtc);
|
||||
Assert.True(restored.ContainsPath("c.jpg"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void InterruptedTransfersResumeAsQueued()
|
||||
{
|
||||
var queue = new UploadQueue();
|
||||
queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }, [Entry("a.cr2", null, null), Entry("b.cr2", null, null)]);
|
||||
queue.Items[0].State = UploadItemState.Uploading;
|
||||
queue.Items[0].Sha256 = new string('b', 64);
|
||||
queue.Items[1].State = UploadItemState.Hashing;
|
||||
|
||||
var path = Path.Combine(context.CreateTempDirectory(), "q.json");
|
||||
QueueStore.Write(path, QueueStore.Serialize(queue));
|
||||
var restored = new UploadQueue();
|
||||
QueueStore.Load(path, restored);
|
||||
|
||||
Assert.Equal(UploadItemState.Queued, restored.Items[0].State);
|
||||
Assert.Equal(UploadItemState.Queued, restored.Items[1].State);
|
||||
// The known hash is kept: the server resumes from the chunks it already holds.
|
||||
Assert.Equal(new string('b', 64), restored.Items[0].Sha256);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CorruptOrMissingFileGivesAnEmptyQueue()
|
||||
{
|
||||
var dir = context.CreateTempDirectory();
|
||||
var corrupt = Path.Combine(dir, "q.json");
|
||||
File.WriteAllText(corrupt, "{ not json");
|
||||
var queue = new UploadQueue();
|
||||
QueueStore.Load(corrupt, queue);
|
||||
QueueStore.Load(Path.Combine(dir, "missing.json"), queue);
|
||||
Assert.Empty(queue.Items);
|
||||
Assert.False(queue.IsPaused);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AddingSkipsPathsAlreadyQueued()
|
||||
{
|
||||
var queue = new UploadQueue();
|
||||
var batch = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions };
|
||||
Assert.Equal(2, queue.Add(batch, [Entry("a.jpg", null, null), Entry("b.jpg", null, null)]));
|
||||
Assert.Equal(1, queue.Add(batch, [Entry("b.jpg", null, null), Entry("c.jpg", null, null)]));
|
||||
Assert.Equal(3, queue.Items.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void FailedItemsComeBackAfterTheDelayUntilRetriesRunOut()
|
||||
{
|
||||
var queue = new UploadQueue();
|
||||
queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions }, [Entry("a.jpg", null, null)]);
|
||||
var item = queue.Items[0];
|
||||
item.State = UploadItemState.Hashing;
|
||||
queue.Fail(item, UploadFailure.FileUnreadable, null, Now);
|
||||
Assert.Null(queue.NextReady(Now.AddSeconds(29)));
|
||||
Assert.Same(item, queue.NextReady(Now + UploadQueue.RetryDelay));
|
||||
Assert.True(queue.HasPending);
|
||||
|
||||
item.Retries = UploadQueue.MaxRetries;
|
||||
queue.Fail(item, UploadFailure.FileUnreadable, null, Now);
|
||||
Assert.Null(item.NextRetryUtc);
|
||||
Assert.Null(queue.NextReady(Now.AddDays(1)));
|
||||
Assert.False(queue.HasPending);
|
||||
|
||||
queue.Requeue(item);
|
||||
Assert.Equal(0, item.Retries);
|
||||
Assert.Same(item, queue.NextReady(Now));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RemovingCompletedItemsKeepsTheRest()
|
||||
{
|
||||
var queue = new UploadQueue();
|
||||
var batch = new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions };
|
||||
queue.Add(batch, [Entry("a.jpg", null, null), Entry("b.jpg", null, null), Entry("c.jpg", null, null)]);
|
||||
queue.Items[0].State = UploadItemState.Verified;
|
||||
queue.Items[2].State = UploadItemState.Skipped;
|
||||
queue.RemoveCompleted();
|
||||
Assert.Equal(1, queue.Items.Count);
|
||||
Assert.False(queue.ContainsPath("a.jpg"));
|
||||
Assert.True(queue.ContainsPath("b.jpg"));
|
||||
Assert.Equal(1, queue.Batches.Count);
|
||||
}
|
||||
|
||||
private static QueueEntry Entry(string name, string? session, string? watchRoot) =>
|
||||
new(name, 1_000 + name.Length, new DateTime(2026, 9, 28, 16, 2, 11, DateTimeKind.Utc), new DateTime(2026, 9, 28, 18, 2, 11), session, watchRoot);
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
using System.Globalization;
|
||||
using Titano.Testing;
|
||||
using Titano.Uploader.Text;
|
||||
using Titano.Uploader.Uploads;
|
||||
|
||||
namespace Titano.App.Tests.Uploader;
|
||||
|
||||
public sealed class UploaderTextTests
|
||||
{
|
||||
private static readonly CultureInfo En = CultureInfo.GetCultureInfo("en");
|
||||
private static readonly CultureInfo It = CultureInfo.GetCultureInfo("it");
|
||||
|
||||
private static readonly QueueSummary Mockup = new(4_812, 96_200_000_000, 2_214, 2_201, 0, 0, 44_000_000_000, "2026-09-28_18-02-11_0413.CR2")
|
||||
{
|
||||
RemainingBytes = 93_480_000_000,
|
||||
};
|
||||
|
||||
[Test]
|
||||
public void QueueLineMatchesTheMockup()
|
||||
{
|
||||
var left = RateMeter.TimeLeft(Mockup.RemainingBytes, 38_000_000);
|
||||
Assert.Equal("4,812 files · 96.2 GB · 38 MB/s · 41 min left", UploaderText.QueueLine(Mockup, QueueActivity.Running, 38_000_000, left, En));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void QueueLineFollowsTheCulture()
|
||||
{
|
||||
Assert.Equal("4.812 file · 96,2 GB · 38 MB/s · mancano 41 min", UploaderText.QueueLine(Mockup, QueueActivity.Running, 38_000_000, TimeSpan.FromMinutes(41), It));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void QueueLineNamesTheActivityWhenNotRunning()
|
||||
{
|
||||
Assert.Equal("4,812 files · 96.2 GB · Paused", UploaderText.QueueLine(Mockup, QueueActivity.Paused, 0, null, En));
|
||||
Assert.Equal("4,812 files · 96.2 GB · Waiting for server", UploaderText.QueueLine(Mockup, QueueActivity.WaitingForServer, 0, null, En));
|
||||
Assert.Equal("1 file · 25 MB · Done", UploaderText.QueueLine(Mockup with { Files = 1, Bytes = 25_000_000 }, QueueActivity.Done, 0, null, En));
|
||||
Assert.Equal("Nothing queued", UploaderText.QueueLine(default, QueueActivity.Idle, 0, null, En));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunningLineOmitsRateAndTimeUntilKnown()
|
||||
{
|
||||
Assert.Equal("4,812 files · 96.2 GB", UploaderText.QueueLine(Mockup, QueueActivity.Running, 0, null, En));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ProgressLineMatchesTheMockup()
|
||||
{
|
||||
Assert.Equal("2,214 of 4,812 sent · 2,201 verified by the server · 0 retries", UploaderText.ProgressLine(Mockup, En));
|
||||
Assert.Equal("2,214 of 4,812 sent · 2,201 verified by the server · 1 retry · 3 failed", UploaderText.ProgressLine(Mockup with { Retries = 1, Failed = 3 }, En));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[Case(0.0, "0 B/s")]
|
||||
[Case(950_000.0, "950 KB/s")]
|
||||
[Case(4_240_000.0, "4.2 MB/s")]
|
||||
[Case(38_400_000.0, "38 MB/s")]
|
||||
[Case(1_260_000_000.0, "1.3 GB/s")]
|
||||
public void RatesUseDecimalUnits(double bytesPerSecond, string expected) =>
|
||||
Assert.Equal(expected, UploaderText.Rate(bytesPerSecond, En));
|
||||
|
||||
[Theory]
|
||||
[Case(40.0, "40 s")]
|
||||
[Case(299.0, "5 min")]
|
||||
[Case(3_590.0, "1 h 00 m")]
|
||||
[Case(7_500.0, "2 h 05 m")]
|
||||
[Case(93_600.0, "1 d 2 h")]
|
||||
public void DurationsAreCompact(double seconds, string expected) =>
|
||||
Assert.Equal(expected, UploaderText.Duration(TimeSpan.FromSeconds(seconds), En));
|
||||
|
||||
[Theory]
|
||||
[Case(999L, "999 B")]
|
||||
[Case(1_536L, "1.5 KB")]
|
||||
[Case(96_200_000_000L, "96.2 GB")]
|
||||
[Case(1_900_000_000_000L, "1.9 TB")]
|
||||
public void SizesUseDecimalUnits(long bytes, string expected) => Assert.Equal(expected, UploaderText.Bytes(bytes, En));
|
||||
|
||||
[Test]
|
||||
public void TimeLeftNeedsAUsableRate()
|
||||
{
|
||||
Assert.Equal(TimeSpan.FromMinutes(41), RateMeter.TimeLeft(93_480_000_000, 38_000_000));
|
||||
Assert.Equal(TimeSpan.Zero, RateMeter.TimeLeft(0, 0));
|
||||
Assert.Null(RateMeter.TimeLeft(1_000, 10));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RateMeterAveragesTheWindowAndDecaysWhenStalled()
|
||||
{
|
||||
var meter = new RateMeter(TimeSpan.FromSeconds(10));
|
||||
Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.Zero));
|
||||
for (var s = 0; s <= 20; s++) meter.Add(4_000_000, TimeSpan.FromSeconds(s));
|
||||
Assert.Approximately(4_000_000, meter.BytesPerSecond(TimeSpan.FromSeconds(20)), 1);
|
||||
// Nothing sent for 20 s: the window holds no transfer any more.
|
||||
Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.FromSeconds(40)));
|
||||
meter.Reset();
|
||||
Assert.Equal(0.0, meter.BytesPerSecond(TimeSpan.FromSeconds(40)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SummaryCountsStatesAndBytes()
|
||||
{
|
||||
var queue = new UploadQueue();
|
||||
queue.Add(new UploadBatch { Id = Guid.CreateVersion7(), Mode = UploadTargetMode.Sessions },
|
||||
Enumerable.Range(0, 5).Select(i => new QueueEntry($"{i}.jpg", 100, DateTime.UtcNow, null, null, null)));
|
||||
queue.Items[0].State = UploadItemState.Verified;
|
||||
queue.Items[1].State = UploadItemState.Skipped;
|
||||
queue.Items[2].State = UploadItemState.Uploading;
|
||||
queue.Items[2].BytesSent = 40;
|
||||
queue.Items[3].State = UploadItemState.Failed;
|
||||
queue.Items[3].Retries = 1;
|
||||
|
||||
var s = QueueSummary.From(queue.Items);
|
||||
Assert.Equal(5, s.Files);
|
||||
Assert.Equal(500L, s.Bytes);
|
||||
Assert.Equal(2, s.Sent);
|
||||
Assert.Equal(2, s.Verified);
|
||||
Assert.Equal(1, s.Failed);
|
||||
Assert.Equal(1, s.Retries);
|
||||
Assert.Equal(240L, s.DoneBytes);
|
||||
Assert.Equal(160L, s.RemainingBytes);
|
||||
Assert.Equal("2.jpg", s.CurrentFile);
|
||||
Assert.Approximately(0.48, s.Progress, 1e-9);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user