From 27f0d73f05bb7e15de47e310556534ae3f4f7b96 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 10:27:24 +0000 Subject: [PATCH] feat(uploader): watch folders with stable-file detection and processed-file memory Polls each folder at its interval with FileSystemWatcher hints on local disks (network shares are polled only); a photo is queued once its size and write time are unchanged in two consecutive checks. Files the server confirmed are remembered by path, size and write time in uploader-watch.json together with each folder's last session. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC --- .../Watching/ProcessedFileMemory.cs | 168 ++++++++++++++++ .../Watching/StableFileTracker.cs | 49 +++++ .../Watching/WatchFolderService.cs | 188 ++++++++++++++++++ .../Uploader/ProcessedFileMemoryTests.cs | 83 ++++++++ .../Uploader/StableFileTrackerTests.cs | 71 +++++++ 5 files changed, 559 insertions(+) create mode 100644 src/Titano.Uploader/Watching/ProcessedFileMemory.cs create mode 100644 src/Titano.Uploader/Watching/StableFileTracker.cs create mode 100644 src/Titano.Uploader/Watching/WatchFolderService.cs create mode 100644 tests/Titano.App.Tests/Uploader/ProcessedFileMemoryTests.cs create mode 100644 tests/Titano.App.Tests/Uploader/StableFileTrackerTests.cs diff --git a/src/Titano.Uploader/Watching/ProcessedFileMemory.cs b/src/Titano.Uploader/Watching/ProcessedFileMemory.cs new file mode 100644 index 00000000..9277ca00 --- /dev/null +++ b/src/Titano.Uploader/Watching/ProcessedFileMemory.cs @@ -0,0 +1,168 @@ +using System.IO; +using System.Text.Json.Serialization; +using System.Text.Json; +using Titano.Core.Diagnostics; +using Titano.Core.IO; +using Titano.Uploader.Sources; + +namespace Titano.Uploader.Watching; + +/// +/// Files of each watch folder the server already confirmed, keyed by path + size + write time, so a poll never +/// uploads them again; plus the last session of each folder so new frames keep joining it. Thread-safe. +/// Persisted in uploader-watch.json. +/// +public sealed class ProcessedFileMemory +{ + public const int Version = 1; + + private readonly Lock _gate = new(); + private readonly Dictionary _folders = new(StringComparer.OrdinalIgnoreCase); + private bool _dirty; + + /// True when something changed since the last . + public bool IsDirty + { + get + { + lock (_gate) return _dirty; + } + } + + public bool Contains(string root, string path, long size, DateTime modifiedUtc) + { + lock (_gate) + { + return _folders.TryGetValue(root, out var f) + && f.Files.TryGetValue(Relative(root, path), out var known) + && known.Size == size && known.ModifiedTicks == modifiedUtc.Ticks; + } + } + + public void Add(string root, string path, long size, DateTime modifiedUtc) + { + lock (_gate) + { + GetOrAdd(root).Files[Relative(root, path)] = new Entry(size, modifiedUtc.Ticks); + _dirty = true; + } + } + + /// Forgets files of that a complete scan no longer found. + public void Prune(string root, IEnumerable presentPaths) + { + lock (_gate) + { + if (!_folders.TryGetValue(root, out var f) || f.Files.Count == 0) return; + var present = new HashSet(presentPaths.Select(p => Relative(root, p)), StringComparer.OrdinalIgnoreCase); + foreach (var key in f.Files.Keys.Where(k => !present.Contains(k)).ToList()) + { + f.Files.Remove(key); + _dirty = true; + } + } + } + + public void RemoveFolder(string root) + { + lock (_gate) + { + if (_folders.Remove(root)) _dirty = true; + } + } + + public SessionContinuation? GetSession(string root) + { + lock (_gate) return _folders.TryGetValue(root, out var f) ? f.Session : null; + } + + public void SetSession(string root, SessionContinuation? session) + { + lock (_gate) + { + GetOrAdd(root).Session = session; + _dirty = true; + } + } + + public byte[] Serialize() + { + lock (_gate) + { + var file = new MemoryFile + { + Version = Version, + Folders = + [ + .. _folders.Select(kv => new FolderRecord( + kv.Key, + kv.Value.Session?.Key, + kv.Value.Session?.LastCapture, + [.. kv.Value.Files.Select(f => new FileRecord(f.Key, f.Value.Size, f.Value.ModifiedTicks))])), + ], + }; + _dirty = false; + return JsonSerializer.SerializeToUtf8Bytes(file, WatchJson.Default.MemoryFile); + } + } + + public void Save(string path) => AtomicFile.WriteAllBytes(path, Serialize()); + + /// Loads ; a missing or corrupt file gives an empty memory. + public static ProcessedFileMemory Load(string path) + { + var memory = new ProcessedFileMemory(); + MemoryFile? file = null; + try + { + if (File.Exists(path)) file = JsonSerializer.Deserialize(File.ReadAllBytes(path), WatchJson.Default.MemoryFile); + } + catch (JsonException ex) + { + Log.Warning("watch", "Watch memory unreadable; starting empty", ex); + } + catch (IOException ex) + { + Log.Warning("watch", "Watch memory unreadable; starting empty", ex); + } + if (file is null || file.Version != Version) return memory; + foreach (var record in file.Folders) + { + var folder = memory.GetOrAdd(record.Root); + if (record.SessionKey is not null && record.LastCapture is { } last) folder.Session = new SessionContinuation(record.SessionKey, last); + foreach (var f in record.Files) folder.Files[f.Path] = new Entry(f.Size, f.Modified); + } + return memory; + } + + private Folder GetOrAdd(string root) + { + if (!_folders.TryGetValue(root, out var f)) _folders[root] = f = new Folder(); + return f; + } + + /// Paths are stored relative to the folder to keep the file small. + private static string Relative(string root, string path) => Path.GetRelativePath(root, path); + + private readonly record struct Entry(long Size, long ModifiedTicks); + + private sealed class Folder + { + public Dictionary Files { get; } = new(StringComparer.OrdinalIgnoreCase); + public SessionContinuation? Session { get; set; } + } + + internal sealed class MemoryFile + { + public int Version { get; set; } + public List Folders { get; set; } = []; + } + + internal sealed record FolderRecord(string Root, string? SessionKey, DateTime? LastCapture, List Files); + + internal sealed record FileRecord(string Path, long Size, long Modified); +} + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(ProcessedFileMemory.MemoryFile))] +internal sealed partial class WatchJson : JsonSerializerContext; diff --git a/src/Titano.Uploader/Watching/StableFileTracker.cs b/src/Titano.Uploader/Watching/StableFileTracker.cs new file mode 100644 index 00000000..b1958744 --- /dev/null +++ b/src/Titano.Uploader/Watching/StableFileTracker.cs @@ -0,0 +1,49 @@ +namespace Titano.Uploader.Watching; + +/// Size and last-write time of a file at one check. +public readonly record struct FileObservation(string Path, long Size, DateTime ModifiedUtc); + +/// +/// Decides when a file in a watch folder has stopped growing: it is reported once its size and write time were +/// identical in consecutive checks, then forgotten (the caller enqueues it). +/// +public sealed class StableFileTracker +{ + public const int RequiredChecks = 2; + + private readonly Dictionary _seen = new(StringComparer.OrdinalIgnoreCase); + + /// Files seen but not yet stable. + public int PendingCount => _seen.Count; + + /// + /// Records a complete scan: returns the files that are now stable and forgets files missing from the scan. + /// + public IReadOnlyList Check(IEnumerable scan) + { + var present = new HashSet(StringComparer.OrdinalIgnoreCase); + var stable = new List(); + foreach (var o in scan) + { + present.Add(o.Path); + if (Observe(o)) stable.Add(o); + } + foreach (var path in _seen.Keys.Where(p => !present.Contains(p)).ToList()) _seen.Remove(path); + return stable; + } + + /// Records one observation; true when the file just became stable. + public bool Observe(FileObservation o) + { + var count = _seen.TryGetValue(o.Path, out var last) && last.Size == o.Size && last.Modified == o.ModifiedUtc ? last.Count + 1 : 1; + if (count >= RequiredChecks) + { + _seen.Remove(o.Path); + return true; + } + _seen[o.Path] = (o.Size, o.ModifiedUtc, count); + return false; + } + + public void Clear() => _seen.Clear(); +} diff --git a/src/Titano.Uploader/Watching/WatchFolderService.cs b/src/Titano.Uploader/Watching/WatchFolderService.cs new file mode 100644 index 00000000..779a3612 --- /dev/null +++ b/src/Titano.Uploader/Watching/WatchFolderService.cs @@ -0,0 +1,188 @@ +using System.IO; +using Titano.Core.Diagnostics; +using Titano.Uploader.Sources; + +namespace Titano.Uploader.Watching; + +/// +/// Polls each watch folder at its interval, with hints on local disks (change +/// notifications are unreliable on network shares, which are only polled). A photo is reported once it is stable +/// (), not yet confirmed by the server () and not queued. +/// +public sealed class WatchFolderService(ProcessedFileMemory memory, Func isQueued) : IDisposable +{ + /// Re-check cadence while some file is still growing. + private static readonly TimeSpan QuickCheck = TimeSpan.FromSeconds(10); + + /// Lets a burst of change notifications settle before scanning. + private static readonly TimeSpan HintDebounce = TimeSpan.FromSeconds(2); + + private readonly Dictionary _folders = new(StringComparer.OrdinalIgnoreCase); + private readonly ProcessedFileMemory _memory = memory; + private readonly Func _isQueued = isQueued; + + /// Stable new photos of a folder; raised on the thread that called . + public event Action>? FilesReady; + + /// A folder became reachable (true) or unreachable (false); raised like . + public event Action? AvailabilityChanged; + + /// UNC paths and mapped network drives. + public static bool IsNetworkPath(string path) + { + if (path.StartsWith(@"\\", StringComparison.Ordinal)) return true; + try + { + var root = Path.GetPathRoot(Path.GetFullPath(path)); + return !string.IsNullOrEmpty(root) && new DriveInfo(root).DriveType == DriveType.Network; + } + catch (Exception ex) when (ex is ArgumentException or IOException) + { + return false; + } + } + + /// 60 s on local disks; 5 min on network shares, where every poll lists the share. + public static TimeSpan DefaultInterval(string path) => IsNetworkPath(path) ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(60); + + /// Starts (or restarts with a new interval) watching . + public void Watch(string root, TimeSpan interval) + { + Stop(root); + var folder = new Folder(root, interval, IsNetworkPath(root), SynchronizationContext.Current, this); + _folders[root] = folder; + folder.Start(); + } + + public void Stop(string root) + { + if (_folders.Remove(root, out var folder)) folder.Dispose(); + } + + public void Dispose() + { + foreach (var f in _folders.Values) f.Dispose(); + _folders.Clear(); + } + + private void Raise(SynchronizationContext? context, Action action) + { + if (context is null) action(); + else context.Post(_ => action(), null); + } + + private sealed class Folder(string root, TimeSpan interval, bool network, SynchronizationContext? context, WatchFolderService owner) : IDisposable + { + private readonly StableFileTracker _tracker = new(); + private readonly CancellationTokenSource _cts = new(); + private FileSystemWatcher? _watcher; + private TaskCompletionSource _hint = NewHint(); + private bool? _available; + + public void Start() => _ = Task.Run(() => LoopAsync(_cts.Token)); + + public void Dispose() + { + _cts.Cancel(); + _watcher?.Dispose(); + _watcher = null; + } + + private async Task LoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + // Replaced before the scan so notifications arriving during it trigger the next one. + Volatile.Write(ref _hint, NewHint()); + var available = Scan(ct); + if (available != _available) + { + _available = available; + if (available && !network) StartWatcher(); + owner.Raise(context, () => owner.AvailabilityChanged?.Invoke(root, available)); + } + var wait = _tracker.PendingCount > 0 && QuickCheck < interval ? QuickCheck : interval; + try + { + var hint = Volatile.Read(ref _hint).Task; + if (await Task.WhenAny(Task.Delay(wait, ct), hint).ConfigureAwait(false) == hint) + await Task.Delay(HintDebounce, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + } + } + + /// One complete listing; false when the folder cannot be reached. + private bool Scan(CancellationToken ct) + { + if (!Directory.Exists(root)) return false; + var present = new List(); + var candidates = new Dictionary(StringComparer.OrdinalIgnoreCase); + try + { + foreach (var file in PhotoScanner.Files(root, ct)) + { + present.Add(file.FullName); + if (IsKnown(file)) continue; + candidates[file.FullName] = file; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A partial listing must not prune the memory or report files. + Log.Info("watch", $"{root} not readable: {ex.Message}"); + return false; + } + catch (OperationCanceledException) + { + return _available ?? false; + } + + owner._memory.Prune(root, present); + var stable = _tracker.Check(candidates.Values.Select(f => new FileObservation(f.FullName, f.Length, f.LastWriteTimeUtc))); + if (stable.Count > 0) + { + var files = stable.Select(o => PhotoScanner.ToSource(candidates[o.Path], root)).ToList(); + Log.Info("watch", $"{files.Count} new file(s) in {root}"); + owner.Raise(context, () => owner.FilesReady?.Invoke(root, files)); + } + return true; + } + + /// Already confirmed by the server, or already in the queue. + private bool IsKnown(FileInfo file) => + owner._memory.Contains(root, file.FullName, file.Length, file.LastWriteTimeUtc) || owner._isQueued(file.FullName); + + private void StartWatcher() + { + _watcher?.Dispose(); + try + { + _watcher = new FileSystemWatcher(root) + { + IncludeSubdirectories = true, + NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.Size | NotifyFilters.LastWrite, + }; + _watcher.Created += (_, _) => Hint(); + _watcher.Changed += (_, _) => Hint(); + _watcher.Renamed += (_, _) => Hint(); + // A lost buffer means unknown changes: scan. + _watcher.Error += (_, _) => Hint(); + _watcher.EnableRaisingEvents = true; + } + catch (Exception ex) when (ex is ArgumentException or IOException or PlatformNotSupportedException) + { + Log.Info("watch", $"No change notifications for {root}; polling only: {ex.Message}"); + _watcher?.Dispose(); + _watcher = null; + } + } + + private void Hint() => Volatile.Read(ref _hint).TrySetResult(); + + private static TaskCompletionSource NewHint() => new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/tests/Titano.App.Tests/Uploader/ProcessedFileMemoryTests.cs b/tests/Titano.App.Tests/Uploader/ProcessedFileMemoryTests.cs new file mode 100644 index 00000000..9cb80990 --- /dev/null +++ b/tests/Titano.App.Tests/Uploader/ProcessedFileMemoryTests.cs @@ -0,0 +1,83 @@ +using System.IO; +using Titano.Testing; +using Titano.Uploader.Sources; +using Titano.Uploader.Watching; + +namespace Titano.App.Tests.Uploader; + +public sealed class ProcessedFileMemoryTests(TestContext context) +{ + private static readonly DateTime Written = new(2026, 9, 28, 16, 2, 11, DateTimeKind.Utc); + + [Test] + public void RemembersPathSizeAndWriteTime() + { + var root = Path.Combine(Path.GetTempPath(), "roof"); + var photo = Path.Combine(root, "100CANON", "IMG_0413.CR2"); + var memory = new ProcessedFileMemory(); + Assert.False(memory.Contains(root, photo, 25_000_000, Written)); + + memory.Add(root, photo, 25_000_000, Written); + Assert.True(memory.IsDirty); + Assert.True(memory.Contains(root, photo, 25_000_000, Written)); + Assert.False(memory.Contains(root, photo, 25_000_001, Written), "a resized file is new"); + Assert.False(memory.Contains(root, photo, 25_000_000, Written.AddSeconds(1)), "a rewritten file is new"); + Assert.False(memory.Contains(Path.Combine(Path.GetTempPath(), "garden"), photo, 25_000_000, Written), "folders are separate"); + } + + [Test] + public void PruneForgetsFilesNoLongerInTheFolder() + { + var root = Path.Combine(Path.GetTempPath(), "roof"); + var kept = Path.Combine(root, "a.jpg"); + var gone = Path.Combine(root, "b.jpg"); + var memory = new ProcessedFileMemory(); + memory.Add(root, kept, 1, Written); + memory.Add(root, gone, 2, Written); + + memory.Prune(root, [kept]); + Assert.True(memory.Contains(root, kept, 1, Written)); + Assert.False(memory.Contains(root, gone, 2, Written)); + } + + [Test] + public void RoundTripKeepsFilesAndTheLastSession() + { + var dir = context.CreateTempDirectory(); + var root = Path.Combine(dir, "cam"); + var photo = Path.Combine(root, "IMG_0001.JPG"); + var memory = new ProcessedFileMemory(); + memory.Add(root, photo, 4_200_000, Written); + memory.SetSession(root, new SessionContinuation("1a2b3c4d-20260928-180000", new DateTime(2026, 9, 28, 18, 40, 0))); + + var path = Path.Combine(dir, "uploader-watch.json"); + memory.Save(path); + Assert.False(memory.IsDirty); + var restored = ProcessedFileMemory.Load(path); + + Assert.True(restored.Contains(root, photo, 4_200_000, Written)); + Assert.Equal(new SessionContinuation("1a2b3c4d-20260928-180000", new DateTime(2026, 9, 28, 18, 40, 0)), restored.GetSession(root)); + Assert.False(restored.IsDirty); + } + + [Test] + public void RemovedFolderIsForgotten() + { + var root = Path.Combine(Path.GetTempPath(), "roof"); + var memory = new ProcessedFileMemory(); + memory.Add(root, Path.Combine(root, "a.jpg"), 1, Written); + memory.SetSession(root, new SessionContinuation("k", Written)); + memory.RemoveFolder(root); + Assert.False(memory.Contains(root, Path.Combine(root, "a.jpg"), 1, Written)); + Assert.Null(memory.GetSession(root)); + } + + [Test] + public void CorruptFileGivesAnEmptyMemory() + { + var path = Path.Combine(context.CreateTempDirectory(), "uploader-watch.json"); + File.WriteAllText(path, "[1,2"); + var memory = ProcessedFileMemory.Load(path); + Assert.False(memory.Contains("x", "x/a.jpg", 1, Written)); + } +} diff --git a/tests/Titano.App.Tests/Uploader/StableFileTrackerTests.cs b/tests/Titano.App.Tests/Uploader/StableFileTrackerTests.cs new file mode 100644 index 00000000..5be3cb8e --- /dev/null +++ b/tests/Titano.App.Tests/Uploader/StableFileTrackerTests.cs @@ -0,0 +1,71 @@ +using Titano.Testing; +using Titano.Uploader.Watching; + +namespace Titano.App.Tests.Uploader; + +public sealed class StableFileTrackerTests +{ + private static readonly DateTime T0 = new(2026, 9, 28, 16, 0, 0, DateTimeKind.Utc); + + [Test] + public void FileIsStableAfterTwoIdenticalChecks() + { + var tracker = new StableFileTracker(); + var photo = new FileObservation("a.cr2", 25_000_000, T0); + Assert.Empty(tracker.Check([photo])); + Assert.Equal(1, tracker.PendingCount); + Assert.SequenceEqual([photo], tracker.Check([photo])); + Assert.Equal(0, tracker.PendingCount); + } + + [Test] + public void GrowingFileWaitsUntilItStopsGrowing() + { + var tracker = new StableFileTracker(); + Assert.Empty(tracker.Check([new("a.cr2", 1_000, T0)])); + Assert.Empty(tracker.Check([new("a.cr2", 9_000, T0.AddSeconds(1))])); + Assert.Empty(tracker.Check([new("a.cr2", 25_000, T0.AddSeconds(2))])); + Assert.Count(1, tracker.Check([new("a.cr2", 25_000, T0.AddSeconds(2))])); + } + + [Test] + public void TouchedFileWithSameSizeRestartsTheCount() + { + var tracker = new StableFileTracker(); + Assert.Empty(tracker.Check([new("a.jpg", 500, T0)])); + Assert.Empty(tracker.Check([new("a.jpg", 500, T0.AddSeconds(5))])); + Assert.Count(1, tracker.Check([new("a.jpg", 500, T0.AddSeconds(5))])); + } + + [Test] + public void FileMissingFromAScanIsForgotten() + { + var tracker = new StableFileTracker(); + var photo = new FileObservation("a.jpg", 500, T0); + Assert.Empty(tracker.Check([photo])); + Assert.Empty(tracker.Check([])); + Assert.Empty(tracker.Check([photo])); + Assert.Count(1, tracker.Check([photo])); + } + + [Test] + public void FilesProgressIndependently() + { + var tracker = new StableFileTracker(); + var done = new FileObservation("done.jpg", 500, T0); + Assert.Empty(tracker.Check([done, new("growing.jpg", 10, T0)])); + var stable = tracker.Check([done, new("growing.jpg", 20, T0.AddSeconds(1))]); + Assert.SequenceEqual([done], stable); + Assert.Equal(1, tracker.PendingCount); + } + + [Test] + public void SingleObservationsUseTheSameRule() + { + var tracker = new StableFileTracker(); + var photo = new FileObservation("a.nef", 42, T0); + Assert.False(tracker.Observe(photo)); + Assert.True(tracker.Observe(photo)); + Assert.False(tracker.Observe(photo)); + } +}