feat(uploader): watch folders with stable-file detection and processed-file memory
Polls each folder at its interval with FileSystemWatcher hints on local disks (network shares are polled only); a photo is queued once its size and write time are unchanged in two consecutive checks. Files the server confirmed are remembered by path, size and write time in uploader-watch.json together with each folder's last session. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
5 files changed
+559
No files matched your search
@@ -0,0 +1,168 @@
|
||||
using System.IO;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Uploader.Sources;
|
||||
|
||||
namespace Titano.Uploader.Watching;
|
||||
|
||||
/// <summary>
|
||||
/// Files of each watch folder the server already confirmed, keyed by path + size + write time, so a poll never
|
||||
/// uploads them again; plus the last session of each folder so new frames keep joining it. Thread-safe.
|
||||
/// Persisted in <c>uploader-watch.json</c>.
|
||||
/// </summary>
|
||||
public sealed class ProcessedFileMemory
|
||||
{
|
||||
public const int Version = 1;
|
||||
|
||||
private readonly Lock _gate = new();
|
||||
private readonly Dictionary<string, Folder> _folders = new(StringComparer.OrdinalIgnoreCase);
|
||||
private bool _dirty;
|
||||
|
||||
/// <summary>True when something changed since the last <see cref="Serialize"/>.</summary>
|
||||
public bool IsDirty
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_gate) return _dirty;
|
||||
}
|
||||
}
|
||||
|
||||
public bool Contains(string root, string path, long size, DateTime modifiedUtc)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
return _folders.TryGetValue(root, out var f)
|
||||
&& f.Files.TryGetValue(Relative(root, path), out var known)
|
||||
&& known.Size == size && known.ModifiedTicks == modifiedUtc.Ticks;
|
||||
}
|
||||
}
|
||||
|
||||
public void Add(string root, string path, long size, DateTime modifiedUtc)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
GetOrAdd(root).Files[Relative(root, path)] = new Entry(size, modifiedUtc.Ticks);
|
||||
_dirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Forgets files of <paramref name="root"/> that a complete scan no longer found.</summary>
|
||||
public void Prune(string root, IEnumerable<string> presentPaths)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_folders.TryGetValue(root, out var f) || f.Files.Count == 0) return;
|
||||
var present = new HashSet<string>(presentPaths.Select(p => Relative(root, p)), StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var key in f.Files.Keys.Where(k => !present.Contains(k)).ToList())
|
||||
{
|
||||
f.Files.Remove(key);
|
||||
_dirty = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void RemoveFolder(string root)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_folders.Remove(root)) _dirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
public SessionContinuation? GetSession(string root)
|
||||
{
|
||||
lock (_gate) return _folders.TryGetValue(root, out var f) ? f.Session : null;
|
||||
}
|
||||
|
||||
public void SetSession(string root, SessionContinuation? session)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
GetOrAdd(root).Session = session;
|
||||
_dirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
public byte[] Serialize()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var file = new MemoryFile
|
||||
{
|
||||
Version = Version,
|
||||
Folders =
|
||||
[
|
||||
.. _folders.Select(kv => new FolderRecord(
|
||||
kv.Key,
|
||||
kv.Value.Session?.Key,
|
||||
kv.Value.Session?.LastCapture,
|
||||
[.. kv.Value.Files.Select(f => new FileRecord(f.Key, f.Value.Size, f.Value.ModifiedTicks))])),
|
||||
],
|
||||
};
|
||||
_dirty = false;
|
||||
return JsonSerializer.SerializeToUtf8Bytes(file, WatchJson.Default.MemoryFile);
|
||||
}
|
||||
}
|
||||
|
||||
public void Save(string path) => AtomicFile.WriteAllBytes(path, Serialize());
|
||||
|
||||
/// <summary>Loads <paramref name="path"/>; a missing or corrupt file gives an empty memory.</summary>
|
||||
public static ProcessedFileMemory Load(string path)
|
||||
{
|
||||
var memory = new ProcessedFileMemory();
|
||||
MemoryFile? file = null;
|
||||
try
|
||||
{
|
||||
if (File.Exists(path)) file = JsonSerializer.Deserialize(File.ReadAllBytes(path), WatchJson.Default.MemoryFile);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
Log.Warning("watch", "Watch memory unreadable; starting empty", ex);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
Log.Warning("watch", "Watch memory unreadable; starting empty", ex);
|
||||
}
|
||||
if (file is null || file.Version != Version) return memory;
|
||||
foreach (var record in file.Folders)
|
||||
{
|
||||
var folder = memory.GetOrAdd(record.Root);
|
||||
if (record.SessionKey is not null && record.LastCapture is { } last) folder.Session = new SessionContinuation(record.SessionKey, last);
|
||||
foreach (var f in record.Files) folder.Files[f.Path] = new Entry(f.Size, f.Modified);
|
||||
}
|
||||
return memory;
|
||||
}
|
||||
|
||||
private Folder GetOrAdd(string root)
|
||||
{
|
||||
if (!_folders.TryGetValue(root, out var f)) _folders[root] = f = new Folder();
|
||||
return f;
|
||||
}
|
||||
|
||||
/// <summary>Paths are stored relative to the folder to keep the file small.</summary>
|
||||
private static string Relative(string root, string path) => Path.GetRelativePath(root, path);
|
||||
|
||||
private readonly record struct Entry(long Size, long ModifiedTicks);
|
||||
|
||||
private sealed class Folder
|
||||
{
|
||||
public Dictionary<string, Entry> Files { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public SessionContinuation? Session { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class MemoryFile
|
||||
{
|
||||
public int Version { get; set; }
|
||||
public List<FolderRecord> Folders { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed record FolderRecord(string Root, string? SessionKey, DateTime? LastCapture, List<FileRecord> Files);
|
||||
|
||||
internal sealed record FileRecord(string Path, long Size, long Modified);
|
||||
}
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(ProcessedFileMemory.MemoryFile))]
|
||||
internal sealed partial class WatchJson : JsonSerializerContext;
|
||||
Reference in new issue
Block a user