feat(uploader): watch folders with stable-file detection and processed-file memory

Polls each folder at its interval with FileSystemWatcher hints on local
disks (network shares are polled only); a photo is queued once its size
and write time are unchanged in two consecutive checks. Files the server
confirmed are remembered by path, size and write time in
uploader-watch.json together with each folder's last session.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 10:27:24 +00:00
1 parent 2a74c12fd1
commit 27f0d73f05
5 files changed
+559

No files matched your search

@@ -0,0 +1,168 @@
using System.IO;
using System.Text.Json.Serialization;
using System.Text.Json;
using Titano.Core.Diagnostics;
using Titano.Core.IO;
using Titano.Uploader.Sources;
namespace Titano.Uploader.Watching;
/// <summary>
/// Files of each watch folder the server already confirmed, keyed by path + size + write time, so a poll never
/// uploads them again; plus the last session of each folder so new frames keep joining it. Thread-safe.
/// Persisted in <c>uploader-watch.json</c>.
/// </summary>
public sealed class ProcessedFileMemory
{
public const int Version = 1;
private readonly Lock _gate = new();
private readonly Dictionary<string, Folder> _folders = new(StringComparer.OrdinalIgnoreCase);
private bool _dirty;
/// <summary>True when something changed since the last <see cref="Serialize"/>.</summary>
public bool IsDirty
{
get
{
lock (_gate) return _dirty;
}
}
public bool Contains(string root, string path, long size, DateTime modifiedUtc)
{
lock (_gate)
{
return _folders.TryGetValue(root, out var f)
&& f.Files.TryGetValue(Relative(root, path), out var known)
&& known.Size == size && known.ModifiedTicks == modifiedUtc.Ticks;
}
}
public void Add(string root, string path, long size, DateTime modifiedUtc)
{
lock (_gate)
{
GetOrAdd(root).Files[Relative(root, path)] = new Entry(size, modifiedUtc.Ticks);
_dirty = true;
}
}
/// <summary>Forgets files of <paramref name="root"/> that a complete scan no longer found.</summary>
public void Prune(string root, IEnumerable<string> presentPaths)
{
lock (_gate)
{
if (!_folders.TryGetValue(root, out var f) || f.Files.Count == 0) return;
var present = new HashSet<string>(presentPaths.Select(p => Relative(root, p)), StringComparer.OrdinalIgnoreCase);
foreach (var key in f.Files.Keys.Where(k => !present.Contains(k)).ToList())
{
f.Files.Remove(key);
_dirty = true;
}
}
}
public void RemoveFolder(string root)
{
lock (_gate)
{
if (_folders.Remove(root)) _dirty = true;
}
}
public SessionContinuation? GetSession(string root)
{
lock (_gate) return _folders.TryGetValue(root, out var f) ? f.Session : null;
}
public void SetSession(string root, SessionContinuation? session)
{
lock (_gate)
{
GetOrAdd(root).Session = session;
_dirty = true;
}
}
public byte[] Serialize()
{
lock (_gate)
{
var file = new MemoryFile
{
Version = Version,
Folders =
[
.. _folders.Select(kv => new FolderRecord(
kv.Key,
kv.Value.Session?.Key,
kv.Value.Session?.LastCapture,
[.. kv.Value.Files.Select(f => new FileRecord(f.Key, f.Value.Size, f.Value.ModifiedTicks))])),
],
};
_dirty = false;
return JsonSerializer.SerializeToUtf8Bytes(file, WatchJson.Default.MemoryFile);
}
}
public void Save(string path) => AtomicFile.WriteAllBytes(path, Serialize());
/// <summary>Loads <paramref name="path"/>; a missing or corrupt file gives an empty memory.</summary>
public static ProcessedFileMemory Load(string path)
{
var memory = new ProcessedFileMemory();
MemoryFile? file = null;
try
{
if (File.Exists(path)) file = JsonSerializer.Deserialize(File.ReadAllBytes(path), WatchJson.Default.MemoryFile);
}
catch (JsonException ex)
{
Log.Warning("watch", "Watch memory unreadable; starting empty", ex);
}
catch (IOException ex)
{
Log.Warning("watch", "Watch memory unreadable; starting empty", ex);
}
if (file is null || file.Version != Version) return memory;
foreach (var record in file.Folders)
{
var folder = memory.GetOrAdd(record.Root);
if (record.SessionKey is not null && record.LastCapture is { } last) folder.Session = new SessionContinuation(record.SessionKey, last);
foreach (var f in record.Files) folder.Files[f.Path] = new Entry(f.Size, f.Modified);
}
return memory;
}
private Folder GetOrAdd(string root)
{
if (!_folders.TryGetValue(root, out var f)) _folders[root] = f = new Folder();
return f;
}
/// <summary>Paths are stored relative to the folder to keep the file small.</summary>
private static string Relative(string root, string path) => Path.GetRelativePath(root, path);
private readonly record struct Entry(long Size, long ModifiedTicks);
private sealed class Folder
{
public Dictionary<string, Entry> Files { get; } = new(StringComparer.OrdinalIgnoreCase);
public SessionContinuation? Session { get; set; }
}
internal sealed class MemoryFile
{
public int Version { get; set; }
public List<FolderRecord> Folders { get; set; } = [];
}
internal sealed record FolderRecord(string Root, string? SessionKey, DateTime? LastCapture, List<FileRecord> Files);
internal sealed record FileRecord(string Path, long Size, long Modified);
}
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(ProcessedFileMemory.MemoryFile))]
internal sealed partial class WatchJson : JsonSerializerContext;
@@ -0,0 +1,49 @@
namespace Titano.Uploader.Watching;
/// <summary>Size and last-write time of a file at one check.</summary>
public readonly record struct FileObservation(string Path, long Size, DateTime ModifiedUtc);
/// <summary>
/// Decides when a file in a watch folder has stopped growing: it is reported once its size and write time were
/// identical in <see cref="RequiredChecks"/> consecutive checks, then forgotten (the caller enqueues it).
/// </summary>
public sealed class StableFileTracker
{
public const int RequiredChecks = 2;
private readonly Dictionary<string, (long Size, DateTime Modified, int Count)> _seen = new(StringComparer.OrdinalIgnoreCase);
/// <summary>Files seen but not yet stable.</summary>
public int PendingCount => _seen.Count;
/// <summary>
/// Records a complete scan: returns the files that are now stable and forgets files missing from the scan.
/// </summary>
public IReadOnlyList<FileObservation> Check(IEnumerable<FileObservation> scan)
{
var present = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var stable = new List<FileObservation>();
foreach (var o in scan)
{
present.Add(o.Path);
if (Observe(o)) stable.Add(o);
}
foreach (var path in _seen.Keys.Where(p => !present.Contains(p)).ToList()) _seen.Remove(path);
return stable;
}
/// <summary>Records one observation; true when the file just became stable.</summary>
public bool Observe(FileObservation o)
{
var count = _seen.TryGetValue(o.Path, out var last) && last.Size == o.Size && last.Modified == o.ModifiedUtc ? last.Count + 1 : 1;
if (count >= RequiredChecks)
{
_seen.Remove(o.Path);
return true;
}
_seen[o.Path] = (o.Size, o.ModifiedUtc, count);
return false;
}
public void Clear() => _seen.Clear();
}
@@ -0,0 +1,188 @@
using System.IO;
using Titano.Core.Diagnostics;
using Titano.Uploader.Sources;
namespace Titano.Uploader.Watching;
/// <summary>
/// Polls each watch folder at its interval, with <see cref="FileSystemWatcher"/> hints on local disks (change
/// notifications are unreliable on network shares, which are only polled). A photo is reported once it is stable
/// (<see cref="StableFileTracker"/>), not yet confirmed by the server (<see cref="ProcessedFileMemory"/>) and not queued.
/// </summary>
public sealed class WatchFolderService(ProcessedFileMemory memory, Func<string, bool> isQueued) : IDisposable
{
/// <summary>Re-check cadence while some file is still growing.</summary>
private static readonly TimeSpan QuickCheck = TimeSpan.FromSeconds(10);
/// <summary>Lets a burst of change notifications settle before scanning.</summary>
private static readonly TimeSpan HintDebounce = TimeSpan.FromSeconds(2);
private readonly Dictionary<string, Folder> _folders = new(StringComparer.OrdinalIgnoreCase);
private readonly ProcessedFileMemory _memory = memory;
private readonly Func<string, bool> _isQueued = isQueued;
/// <summary>Stable new photos of a folder; raised on the thread that called <see cref="Watch"/>.</summary>
public event Action<string, IReadOnlyList<SourceFile>>? FilesReady;
/// <summary>A folder became reachable (true) or unreachable (false); raised like <see cref="FilesReady"/>.</summary>
public event Action<string, bool>? AvailabilityChanged;
/// <summary>UNC paths and mapped network drives.</summary>
public static bool IsNetworkPath(string path)
{
if (path.StartsWith(@"\\", StringComparison.Ordinal)) return true;
try
{
var root = Path.GetPathRoot(Path.GetFullPath(path));
return !string.IsNullOrEmpty(root) && new DriveInfo(root).DriveType == DriveType.Network;
}
catch (Exception ex) when (ex is ArgumentException or IOException)
{
return false;
}
}
/// <summary>60 s on local disks; 5 min on network shares, where every poll lists the share.</summary>
public static TimeSpan DefaultInterval(string path) => IsNetworkPath(path) ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(60);
/// <summary>Starts (or restarts with a new interval) watching <paramref name="root"/>.</summary>
public void Watch(string root, TimeSpan interval)
{
Stop(root);
var folder = new Folder(root, interval, IsNetworkPath(root), SynchronizationContext.Current, this);
_folders[root] = folder;
folder.Start();
}
public void Stop(string root)
{
if (_folders.Remove(root, out var folder)) folder.Dispose();
}
public void Dispose()
{
foreach (var f in _folders.Values) f.Dispose();
_folders.Clear();
}
private void Raise(SynchronizationContext? context, Action action)
{
if (context is null) action();
else context.Post(_ => action(), null);
}
private sealed class Folder(string root, TimeSpan interval, bool network, SynchronizationContext? context, WatchFolderService owner) : IDisposable
{
private readonly StableFileTracker _tracker = new();
private readonly CancellationTokenSource _cts = new();
private FileSystemWatcher? _watcher;
private TaskCompletionSource _hint = NewHint();
private bool? _available;
public void Start() => _ = Task.Run(() => LoopAsync(_cts.Token));
public void Dispose()
{
_cts.Cancel();
_watcher?.Dispose();
_watcher = null;
}
private async Task LoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
// Replaced before the scan so notifications arriving during it trigger the next one.
Volatile.Write(ref _hint, NewHint());
var available = Scan(ct);
if (available != _available)
{
_available = available;
if (available && !network) StartWatcher();
owner.Raise(context, () => owner.AvailabilityChanged?.Invoke(root, available));
}
var wait = _tracker.PendingCount > 0 && QuickCheck < interval ? QuickCheck : interval;
try
{
var hint = Volatile.Read(ref _hint).Task;
if (await Task.WhenAny(Task.Delay(wait, ct), hint).ConfigureAwait(false) == hint)
await Task.Delay(HintDebounce, ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
}
}
/// <summary>One complete listing; false when the folder cannot be reached.</summary>
private bool Scan(CancellationToken ct)
{
if (!Directory.Exists(root)) return false;
var present = new List<string>();
var candidates = new Dictionary<string, FileInfo>(StringComparer.OrdinalIgnoreCase);
try
{
foreach (var file in PhotoScanner.Files(root, ct))
{
present.Add(file.FullName);
if (IsKnown(file)) continue;
candidates[file.FullName] = file;
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// A partial listing must not prune the memory or report files.
Log.Info("watch", $"{root} not readable: {ex.Message}");
return false;
}
catch (OperationCanceledException)
{
return _available ?? false;
}
owner._memory.Prune(root, present);
var stable = _tracker.Check(candidates.Values.Select(f => new FileObservation(f.FullName, f.Length, f.LastWriteTimeUtc)));
if (stable.Count > 0)
{
var files = stable.Select(o => PhotoScanner.ToSource(candidates[o.Path], root)).ToList();
Log.Info("watch", $"{files.Count} new file(s) in {root}");
owner.Raise(context, () => owner.FilesReady?.Invoke(root, files));
}
return true;
}
/// <summary>Already confirmed by the server, or already in the queue.</summary>
private bool IsKnown(FileInfo file) =>
owner._memory.Contains(root, file.FullName, file.Length, file.LastWriteTimeUtc) || owner._isQueued(file.FullName);
private void StartWatcher()
{
_watcher?.Dispose();
try
{
_watcher = new FileSystemWatcher(root)
{
IncludeSubdirectories = true,
NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.Size | NotifyFilters.LastWrite,
};
_watcher.Created += (_, _) => Hint();
_watcher.Changed += (_, _) => Hint();
_watcher.Renamed += (_, _) => Hint();
// A lost buffer means unknown changes: scan.
_watcher.Error += (_, _) => Hint();
_watcher.EnableRaisingEvents = true;
}
catch (Exception ex) when (ex is ArgumentException or IOException or PlatformNotSupportedException)
{
Log.Info("watch", $"No change notifications for {root}; polling only: {ex.Message}");
_watcher?.Dispose();
_watcher = null;
}
}
private void Hint() => Volatile.Read(ref _hint).TrySetResult();
private static TaskCompletionSource NewHint() => new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
@@ -0,0 +1,83 @@
using System.IO;
using Titano.Testing;
using Titano.Uploader.Sources;
using Titano.Uploader.Watching;
namespace Titano.App.Tests.Uploader;
public sealed class ProcessedFileMemoryTests(TestContext context)
{
private static readonly DateTime Written = new(2026, 9, 28, 16, 2, 11, DateTimeKind.Utc);
[Test]
public void RemembersPathSizeAndWriteTime()
{
var root = Path.Combine(Path.GetTempPath(), "roof");
var photo = Path.Combine(root, "100CANON", "IMG_0413.CR2");
var memory = new ProcessedFileMemory();
Assert.False(memory.Contains(root, photo, 25_000_000, Written));
memory.Add(root, photo, 25_000_000, Written);
Assert.True(memory.IsDirty);
Assert.True(memory.Contains(root, photo, 25_000_000, Written));
Assert.False(memory.Contains(root, photo, 25_000_001, Written), "a resized file is new");
Assert.False(memory.Contains(root, photo, 25_000_000, Written.AddSeconds(1)), "a rewritten file is new");
Assert.False(memory.Contains(Path.Combine(Path.GetTempPath(), "garden"), photo, 25_000_000, Written), "folders are separate");
}
[Test]
public void PruneForgetsFilesNoLongerInTheFolder()
{
var root = Path.Combine(Path.GetTempPath(), "roof");
var kept = Path.Combine(root, "a.jpg");
var gone = Path.Combine(root, "b.jpg");
var memory = new ProcessedFileMemory();
memory.Add(root, kept, 1, Written);
memory.Add(root, gone, 2, Written);
memory.Prune(root, [kept]);
Assert.True(memory.Contains(root, kept, 1, Written));
Assert.False(memory.Contains(root, gone, 2, Written));
}
[Test]
public void RoundTripKeepsFilesAndTheLastSession()
{
var dir = context.CreateTempDirectory();
var root = Path.Combine(dir, "cam");
var photo = Path.Combine(root, "IMG_0001.JPG");
var memory = new ProcessedFileMemory();
memory.Add(root, photo, 4_200_000, Written);
memory.SetSession(root, new SessionContinuation("1a2b3c4d-20260928-180000", new DateTime(2026, 9, 28, 18, 40, 0)));
var path = Path.Combine(dir, "uploader-watch.json");
memory.Save(path);
Assert.False(memory.IsDirty);
var restored = ProcessedFileMemory.Load(path);
Assert.True(restored.Contains(root, photo, 4_200_000, Written));
Assert.Equal(new SessionContinuation("1a2b3c4d-20260928-180000", new DateTime(2026, 9, 28, 18, 40, 0)), restored.GetSession(root));
Assert.False(restored.IsDirty);
}
[Test]
public void RemovedFolderIsForgotten()
{
var root = Path.Combine(Path.GetTempPath(), "roof");
var memory = new ProcessedFileMemory();
memory.Add(root, Path.Combine(root, "a.jpg"), 1, Written);
memory.SetSession(root, new SessionContinuation("k", Written));
memory.RemoveFolder(root);
Assert.False(memory.Contains(root, Path.Combine(root, "a.jpg"), 1, Written));
Assert.Null(memory.GetSession(root));
}
[Test]
public void CorruptFileGivesAnEmptyMemory()
{
var path = Path.Combine(context.CreateTempDirectory(), "uploader-watch.json");
File.WriteAllText(path, "[1,2");
var memory = ProcessedFileMemory.Load(path);
Assert.False(memory.Contains("x", "x/a.jpg", 1, Written));
}
}
@@ -0,0 +1,71 @@
using Titano.Testing;
using Titano.Uploader.Watching;
namespace Titano.App.Tests.Uploader;
public sealed class StableFileTrackerTests
{
private static readonly DateTime T0 = new(2026, 9, 28, 16, 0, 0, DateTimeKind.Utc);
[Test]
public void FileIsStableAfterTwoIdenticalChecks()
{
var tracker = new StableFileTracker();
var photo = new FileObservation("a.cr2", 25_000_000, T0);
Assert.Empty(tracker.Check([photo]));
Assert.Equal(1, tracker.PendingCount);
Assert.SequenceEqual([photo], tracker.Check([photo]));
Assert.Equal(0, tracker.PendingCount);
}
[Test]
public void GrowingFileWaitsUntilItStopsGrowing()
{
var tracker = new StableFileTracker();
Assert.Empty(tracker.Check([new("a.cr2", 1_000, T0)]));
Assert.Empty(tracker.Check([new("a.cr2", 9_000, T0.AddSeconds(1))]));
Assert.Empty(tracker.Check([new("a.cr2", 25_000, T0.AddSeconds(2))]));
Assert.Count(1, tracker.Check([new("a.cr2", 25_000, T0.AddSeconds(2))]));
}
[Test]
public void TouchedFileWithSameSizeRestartsTheCount()
{
var tracker = new StableFileTracker();
Assert.Empty(tracker.Check([new("a.jpg", 500, T0)]));
Assert.Empty(tracker.Check([new("a.jpg", 500, T0.AddSeconds(5))]));
Assert.Count(1, tracker.Check([new("a.jpg", 500, T0.AddSeconds(5))]));
}
[Test]
public void FileMissingFromAScanIsForgotten()
{
var tracker = new StableFileTracker();
var photo = new FileObservation("a.jpg", 500, T0);
Assert.Empty(tracker.Check([photo]));
Assert.Empty(tracker.Check([]));
Assert.Empty(tracker.Check([photo]));
Assert.Count(1, tracker.Check([photo]));
}
[Test]
public void FilesProgressIndependently()
{
var tracker = new StableFileTracker();
var done = new FileObservation("done.jpg", 500, T0);
Assert.Empty(tracker.Check([done, new("growing.jpg", 10, T0)]));
var stable = tracker.Check([done, new("growing.jpg", 20, T0.AddSeconds(1))]);
Assert.SequenceEqual([done], stable);
Assert.Equal(1, tracker.PendingCount);
}
[Test]
public void SingleObservationsUseTheSameRule()
{
var tracker = new StableFileTracker();
var photo = new FileObservation("a.nef", 42, T0);
Assert.False(tracker.Observe(photo));
Assert.True(tracker.Observe(photo));
Assert.False(tracker.Observe(photo));
}
}