diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 20ad648e..a5952f47 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -5,4 +5,26 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 No archive yet + Another Titano ({0}) is writing here. Try again shortly.long: error sentence + The copy of {0} does not match the original.long: error sentence + {0} could not be added to the sequence.long: error sentence + {0} was written by a newer Titano (format {1}).long: error sentence + The archive {0} is offline.long: error sentence + The archive {0} is read-only.long: error sentence + {0} changed after the plan was made.long: error sentence + The card or folder {0} is no longer available.long: error sentence + The archive changed after this plan was made.long: error sentence + Dry run: nothing was changed.long: job log line + Not indexed: {0} ({1})long: job log line + Changed: {0}long: job log line + Damaged: {0}long: job log line + Missing: {0}long: job log line + {0}: {1} made, {2} reused, {3} unreadablelong: job log line + {0} copied, {1} skipped, {2} failedlong: job log line + Not copied: {0} ({1})long: job log line + {0} stale locks, {1} temporary files, {2} packs compacted, {3} duplicate flags changedlong: job log line + {0} unindexed photos, {1} missing photoslong: job log line + {0} folders, {1} indexed, {2} adopted, {3} removedlong: job log line + Sequence not found: {0}long: job log line + {0} photos checked, {1} intact, {2} hashed, {3} with problemslong: job log line diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 00b325d1..62523790 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -5,4 +5,26 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 Nessun archivio + Un altro Titano ({0}) sta scrivendo qui. Riprova tra poco. + La copia di {0} non corrisponde all'originale. + Impossibile aggiungere {0} alla sequenza. + {0} è stato scritto da un Titano più recente (formato {1}). + L'archivio {0} non è raggiungibile. + L'archivio {0} è di sola lettura. + {0} è cambiato dopo la creazione del piano. + La scheda o cartella {0} non è più disponibile. + L'archivio è cambiato dopo la creazione del piano. + Prova: nulla è stato modificato. + Non indicizzata: {0} ({1}) + Modificato: {0} + Danneggiato: {0} + Mancante: {0} + {0}: {1} create, {2} riusate, {3} illeggibili + {0} copiate, {1} saltate, {2} non riuscite + Non copiata: {0} ({1}) + {0} blocchi scaduti, {1} file temporanei, {2} pacchetti compattati, {3} segni di duplicato cambiati + {0} foto non indicizzate, {1} foto mancanti + {0} cartelle, {1} indicizzate, {2} adottate, {3} rimosse + Sequenza non trovata: {0} + {0} foto controllate, {1} integre, {2} con impronta nuova, {3} con problemi diff --git a/src/Titano.Core/Archiving/ArchiveJson.cs b/src/Titano.Core/Archiving/ArchiveJson.cs new file mode 100644 index 00000000..6a893b97 --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveJson.cs @@ -0,0 +1,46 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Titano.Core.Archiving; + +/// Source-generated metadata for every JSON file of the archive layout. +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = true, + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip)] +[JsonSerializable(typeof(ArchiveManifest))] +[JsonSerializable(typeof(SequenceManifest))] +[JsonSerializable(typeof(LeaseRecord))] +[JsonSerializable(typeof(IngestJournal))] +[JsonSerializable(typeof(RenameJournal))] +internal sealed partial class ArchiveJson : JsonSerializerContext; + +/// Indented output for the small files people may open (archive.json, rename journals). +[JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(ArchiveManifest))] +[JsonSerializable(typeof(RenameJournal))] +internal sealed partial class ArchiveJsonIndented : JsonSerializerContext; + +/// +/// Writes enum names and reads unknown names (written by a newer build) or numbers as the default value instead of +/// failing, so a sidecar from a newer Titano is still readable. +/// +public sealed class TolerantEnumConverter : JsonConverter + where TEnum : struct, Enum +{ + public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String && Enum.TryParse(reader.GetString(), ignoreCase: true, out var value) + && Enum.IsDefined(value)) + return value; + if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) reader.Skip(); + return default; + } + + public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToString()); +} diff --git a/src/Titano.Core/Archiving/ArchiveLayout.cs b/src/Titano.Core/Archiving/ArchiveLayout.cs new file mode 100644 index 00000000..ab51614d --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveLayout.cs @@ -0,0 +1,104 @@ +using System.Globalization; +using Titano.Core.IO; + +namespace Titano.Core.Archiving; + +/// Names of the fixed files and folders of the archive layout (BRIEF §7). +public static class ArchiveLayout +{ + /// Metadata folder at the archive root and inside every sequence folder. + public const string MetaFolder = ".titano"; + + public const string ArchiveManifestFile = "archive.json"; + public const string IndexSnapshotFile = "index-snapshot.bin"; + public const string LocksFolder = "locks"; + + /// Verified copies waiting to be committed; on the archive volume so commits are renames. + public const string StagingFolder = "staging"; + + /// Resumable plans of mass operations (rename). + public const string JournalFolder = "journal"; + + public const string OriginalsFolder = "originals"; + public const string ProxiesFolder = "proxies"; + public const string ExportsFolder = "exports"; + public const string SequenceManifestFile = "sequence.json"; + public const string AnalysisFile = "analysis.json"; + public const string ProjectFile = "project.json"; + public const string ThumbsPackFile = "thumbs.pack"; + + /// Moves of an ingest batch that were started but not yet recorded in . + public const string IngestJournalFile = "ingest.pending.json"; + + /// Folder names inside a sequence folder that never hold a sequence of their own. + public static bool IsReservedFolder(string name) => + name.Equals(MetaFolder, StringComparison.OrdinalIgnoreCase) + || name.Equals(OriginalsFolder, StringComparison.OrdinalIgnoreCase) + || name.Equals(ProxiesFolder, StringComparison.OrdinalIgnoreCase) + || name.Equals(ExportsFolder, StringComparison.OrdinalIgnoreCase); + + /// 0001.jpg for frame index 0: proxies are numbered from 1 like the frames of a render. + public static string ProxyFileName(int index) => (index + 1).ToString("0000", CultureInfo.InvariantCulture) + ".jpg"; +} + +/// Absolute paths of one archive root. +public sealed class ArchivePaths +{ + public ArchivePaths(string root) + { + ArgumentException.ThrowIfNullOrWhiteSpace(root); + Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); + MetaDirectory = Path.Combine(Root, ArchiveLayout.MetaFolder); + } + + public string Root { get; } + public string MetaDirectory { get; } + public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.ArchiveManifestFile); + public string SnapshotPath => Path.Combine(MetaDirectory, ArchiveLayout.IndexSnapshotFile); + public string LocksDirectory => Path.Combine(MetaDirectory, ArchiveLayout.LocksFolder); + public string StagingDirectory => Path.Combine(MetaDirectory, ArchiveLayout.StagingFolder); + public string JournalDirectory => Path.Combine(MetaDirectory, ArchiveLayout.JournalFolder); + + /// Paths of the sequence stored in (forward slashes). + public SequencePaths Sequence(string relativeFolder) => new(this, relativeFolder); + + /// Archive-relative path with forward slashes, as stored in the index. + public string ToRelative(string path) => LongPath.ToPortableRelative(Root, path); + + public string FromRelative(string relative) => LongPath.FromPortableRelative(Root, relative); +} + +/// Absolute paths of one sequence folder. +public sealed class SequencePaths +{ + internal SequencePaths(ArchivePaths archive, string relativeFolder) + { + ArgumentNullException.ThrowIfNull(relativeFolder); + RelativeFolder = relativeFolder.Replace('\\', '/').Trim('/'); + Folder = archive.FromRelative(RelativeFolder); + MetaDirectory = Path.Combine(Folder, ArchiveLayout.MetaFolder); + } + + /// Folder relative to the archive root, forward slashes; empty when the root itself holds the photos (adopted folder). + public string RelativeFolder { get; } + + public string Folder { get; } + public string MetaDirectory { get; } + public string Originals => Path.Combine(Folder, ArchiveLayout.OriginalsFolder); + public string Proxies => Path.Combine(Folder, ArchiveLayout.ProxiesFolder); + public string Exports => Path.Combine(Folder, ArchiveLayout.ExportsFolder); + public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.SequenceManifestFile); + public string AnalysisPath => Path.Combine(MetaDirectory, ArchiveLayout.AnalysisFile); + public string ProjectPath => Path.Combine(MetaDirectory, ArchiveLayout.ProjectFile); + public string ThumbsPackPath => Path.Combine(MetaDirectory, ArchiveLayout.ThumbsPackFile); + public string IngestJournalPath => Path.Combine(MetaDirectory, ArchiveLayout.IngestJournalFile); + + public string Proxy(int index) => Path.Combine(Proxies, ArchiveLayout.ProxyFileName(index)); + + /// Absolute path of a frame file given its sidecar path (relative to the sequence folder). + public string File(string sequenceRelative) => LongPath.FromPortableRelative(Folder, sequenceRelative); + + /// Archive-relative path of a frame file (as stored in the frames table). + public string ArchiveRelative(string sequenceRelative) => + RelativeFolder.Length == 0 ? sequenceRelative : RelativeFolder + "/" + sequenceRelative; +} diff --git a/src/Titano.Core/Archiving/ArchiveManifest.cs b/src/Titano.Core/Archiving/ArchiveManifest.cs new file mode 100644 index 00000000..6918ff9f --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveManifest.cs @@ -0,0 +1,49 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Naming; + +namespace Titano.Core.Archiving; + +/// +/// .titano/archive.json at the archive root (BRIEF §7). Fields this build does not know are kept in +/// and written back unchanged. +/// +public sealed record ArchiveManifest +{ + public const int CurrentSchemaVersion = 1; + + public int SchemaVersion { get; init; } = CurrentSchemaVersion; + public required Guid Id { get; init; } + public required string Name { get; init; } + public DateTimeOffset Created { get; init; } + public LayoutTemplates LayoutTemplates { get; init; } = new(); + public ProxyPolicy ProxyPolicy { get; init; } = new(); + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// Naming templates used when frames enter the archive (BRIEF §7.1). +public sealed record LayoutTemplates +{ + public string Folder { get; init; } = TemplateDefaults.Folder; + public string File { get; init; } = TemplateDefaults.File; + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// Proxy generation policy (BRIEF §8.5). +public sealed record ProxyPolicy +{ + public const int Disabled = -1; + public const int FullSize = 0; + + /// Long edge in pixels: 1280, 1920 (default) or 2560; or . + public int LongEdge { get; init; } = 1920; + + public int Quality { get; init; } = 92; + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} diff --git a/src/Titano.Core/Archiving/ArchiveScanner.cs b/src/Titano.Core/Archiving/ArchiveScanner.cs new file mode 100644 index 00000000..47f20980 --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveScanner.cs @@ -0,0 +1,77 @@ +using Titano.Core.Imaging; + +namespace Titano.Core.Archiving; + +/// A folder that holds a sequence: it has a sidecar, or photos to adopt. +/// Archive-relative, forward slashes; empty for the root. +/// Stamp of .titano/sequence.json, null when it has none yet. +/// Stamp of the photo directory (its time changes when files are added or removed). +public sealed record SequenceFolder(string RelativeFolder, FileStamp? Sidecar, FileStamp? Photos); + +/// +/// Walks an archive tree (BRIEF §7). No file-system watcher is involved: callers compare the stamps with those of +/// the previous scan. Hidden folders, symbolic links and the reserved sequence subfolders are not entered. +/// +public static class ArchiveScanner +{ + private const int MaxDepth = 16; + + private static readonly EnumerationOptions Listing = new() + { + IgnoreInaccessible = true, + AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System, + RecurseSubdirectories = false, + }; + + /// Every sequence folder under the root, ordered by relative path (ordinal) so job checkpoints are stable. + public static IReadOnlyList FindSequenceFolders(ArchivePaths archive, CancellationToken cancellationToken = default) + { + var found = new List(); + var pending = new Stack<(string Path, int Depth)>(); + pending.Push((archive.Root, 0)); + while (pending.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var (path, depth) = pending.Pop(); + var relative = archive.ToRelative(path); + var sequence = archive.Sequence(relative == "." ? string.Empty : relative); + var sidecar = FileStamp.Of(sequence.ManifestPath); + var photoDirectory = PhotoDirectory(sequence); + if (sidecar is not null || EnumeratePhotos(photoDirectory).Any()) + found.Add(new SequenceFolder(sequence.RelativeFolder, sidecar, FileStamp.Of(photoDirectory))); + if (depth >= MaxDepth) continue; + foreach (var child in Directory.EnumerateDirectories(path, "*", Listing)) + { + var name = Path.GetFileName(child); + if (!IsHidden(child, name) && !ArchiveLayout.IsReservedFolder(name)) pending.Push((child, depth + 1)); + } + } + found.Sort((a, b) => string.CompareOrdinal(a.RelativeFolder, b.RelativeFolder)); + return found; + } + + /// originals/ when the folder follows the layout, otherwise the folder itself (adopted in place). + public static string PhotoDirectory(SequencePaths sequence) => Directory.Exists(sequence.Originals) ? sequence.Originals : sequence.Folder; + + /// Photos of a sequence folder as sidecar paths (relative to the sequence folder), ordinal order. + public static IReadOnlyList ListPhotos(SequencePaths sequence) + { + var directory = PhotoDirectory(sequence); + return EnumeratePhotos(directory) + .Select(path => Path.GetRelativePath(sequence.Folder, path).Replace('\\', '/')) + .Order(StringComparer.Ordinal) + .ToList(); + } + + /// True for files Titano imports as photos (AppleDouble ._ companions excluded). + public static bool IsPhoto(string fileName) => + !fileName.StartsWith("._", StringComparison.Ordinal) && ImageFormatInfo.IsSupportedExtension(fileName); + + private static IEnumerable EnumeratePhotos(string directory) => + Directory.Exists(directory) + ? Directory.EnumerateFiles(directory, "*", Listing).Where(f => IsPhoto(Path.GetFileName(f)) && !IsHidden(f, Path.GetFileName(f))) + : []; + + private static bool IsHidden(string path, string name) => + name.StartsWith('.') || (OperatingSystem.IsWindows() && File.GetAttributes(path).HasFlag(FileAttributes.Hidden)); +} diff --git a/src/Titano.Core/Archiving/ArchiveVolume.cs b/src/Titano.Core/Archiving/ArchiveVolume.cs new file mode 100644 index 00000000..d19a22a4 --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveVolume.cs @@ -0,0 +1,55 @@ +using Titano.Core.Localization; + +namespace Titano.Core.Archiving; + +/// The archive cannot be reached (or written) right now; nothing was changed. +public sealed class ArchiveOfflineException(string root, ArchiveAvailability availability) + : IOException(Loc.L(availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", root)) +{ + public string Root { get; } = root; + public ArchiveAvailability Availability { get; } = availability; +} + +public enum ArchiveAvailability +{ + Online, + + /// Readable but not writable (read-only share, write-protected disk): browse only. + ReadOnly, + + /// Root unreachable (unplugged drive, disconnected share) or no longer an archive. + Offline, +} + +/// Offline detection for archives on removable and network volumes (BRIEF §7). +public static class ArchiveVolume +{ + /// Throws unless the archive is online (writable). + public static void EnsureWritable(ArchivePaths archive) + { + var availability = Probe(archive); + if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(archive.Root, availability); + } + + /// Checks that archive.json is reachable and that the metadata folder accepts writes. + public static ArchiveAvailability Probe(ArchivePaths archive) + { + try + { + if (!File.Exists(archive.ManifestPath)) return ArchiveAvailability.Offline; + var probe = Path.Combine(archive.MetaDirectory, ".probe-" + Guid.NewGuid().ToString("N")[..8]); + using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1, FileOptions.DeleteOnClose)) + { + } + return ArchiveAvailability.Online; + } + catch (UnauthorizedAccessException) + { + return ArchiveAvailability.ReadOnly; + } + catch (IOException) + { + return File.Exists(archive.ManifestPath) ? ArchiveAvailability.ReadOnly : ArchiveAvailability.Offline; + } + } +} diff --git a/src/Titano.Core/Archiving/ContentHash.cs b/src/Titano.Core/Archiving/ContentHash.cs new file mode 100644 index 00000000..ea647a8c --- /dev/null +++ b/src/Titano.Core/Archiving/ContentHash.cs @@ -0,0 +1,44 @@ +using System.Security.Cryptography; + +namespace Titano.Core.Archiving; + +/// SHA-256 of files, streamed (originals can be hundreds of megabytes). +public static class ContentHash +{ + internal const int BufferSize = 1 << 20; + + /// Lower-case hex SHA-256 of the file. + /// Receives the number of bytes read since the previous call. + public static async Task Sha256Async(string path, Action? progress = null, CancellationToken cancellationToken = default) + { + await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, BufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); + return await Sha256Async(stream, progress, cancellationToken).ConfigureAwait(false); + } + + public static async Task Sha256Async(Stream stream, Action? progress = null, CancellationToken cancellationToken = default) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var buffer = new byte[BufferSize]; + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + hash.AppendData(buffer, 0, read); + progress?.Invoke(read); + } + return Convert.ToHexStringLower(hash.GetHashAndReset()); + } +} + +/// Size and modification time of a file, the cheap change detector of incremental scans (BRIEF §7). +public readonly record struct FileStamp(long Size, DateTime ModifiedUtc) +{ + /// The stamp of a file or directory, or null when it does not exist. + public static FileStamp? Of(string path) + { + var file = new FileInfo(path); + if (file.Exists) return new FileStamp(file.Length, file.LastWriteTimeUtc); + var directory = new DirectoryInfo(path); + return directory.Exists ? new FileStamp(0, directory.LastWriteTimeUtc) : null; + } +} diff --git a/src/Titano.Core/Archiving/FrameImageRenderer.cs b/src/Titano.Core/Archiving/FrameImageRenderer.cs new file mode 100644 index 00000000..0cdbfff3 --- /dev/null +++ b/src/Titano.Core/Archiving/FrameImageRenderer.cs @@ -0,0 +1,97 @@ +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; + +namespace Titano.Core.Archiving; + +/// A JPEG made from a frame. +public sealed record EncodedImage(byte[] Jpeg, int Width, int Height, DecodeTier Tier); + +/// +/// Makes thumbnails and proxies of frames (BRIEF §8.5): the embedded preview or the managed JPEG decoder first, then +/// the host's decode providers (WIC on Windows); orientation applied, resampled in linear light, encoded in-house. +/// +public sealed class FrameImageRenderer(ImageDecoder decoder) +{ + public const int ThumbnailLongEdge = 320; + public const int ThumbnailQuality = 80; + + /// A 320 px q80 thumbnail, or null when no decoder can read the frame. + public EncodedImage? Thumbnail(SequencePaths sequence, SidecarFrame frame, CancellationToken cancellationToken = default) + { + using var decoded = Decode(sequence.File(frame.File), frame, ThumbnailLongEdge, DecodePurpose.Thumbnail, preferPreview: true, cancellationToken); + if (decoded is null) return null; + using var small = Resampler.Thumbnail(decoded.Pixels, ThumbnailLongEdge); + return Encode(small, new JpegEncodeOptions { Quality = ThumbnailQuality }, decoded.Tier); + } + + /// A proxy at the policy's long edge and quality with capture time and camera in its Exif; null when undecodable. + public EncodedImage? Proxy(SequencePaths sequence, SidecarFrame frame, SidecarCamera? camera, ProxyPolicy policy, CancellationToken cancellationToken = default) + { + if (policy.LongEdge == ProxyPolicy.Disabled) throw new InvalidOperationException("Proxies are disabled for this archive."); + using var decoded = Decode(sequence.File(frame.File), frame, policy.LongEdge, DecodePurpose.Proxy, preferPreview: false, cancellationToken); + if (decoded is null) return null; + var source = decoded.Pixels; + var (w, h) = policy.LongEdge == ProxyPolicy.FullSize ? (source.Width, source.Height) : Resampler.FitWithin(source.Width, source.Height, policy.LongEdge); + using var resized = Resampler.Resize(source, w, h); + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata + { + Make = camera?.Make, + Model = camera?.Model, + BodySerial = camera?.Serial, + CapturedAt = frame.CapturedAt, + UtcOffsetMinutes = frame.UtcOffset, + ExposureTime = frame.Exposure, + FNumber = frame.FNumber, + Iso = frame.Iso, + FocalLength = frame.Focal, + ExposureBias = frame.Bias, + Software = "Titano", + }); + // The encoder takes the APP1 payload; ExifWriter returns the whole segment (marker and length first). + return Encode(resized, new JpegEncodeOptions { Quality = policy.Quality, App1Segments = [exif[4..]] }, decoded.Tier); + } + + private static EncodedImage Encode(ImageBuffer pixels, JpegEncodeOptions options, DecodeTier tier) => + new(JpegEncoder.Encode(pixels, options), pixels.Width, pixels.Height, tier); + + private DecodedFrame? Decode(string path, SidecarFrame frame, int maxLongEdge, DecodePurpose purpose, bool preferPreview, CancellationToken cancellationToken) + { + if (frame.Format == ImageFileFormat.Jpeg) return DecodeJpeg(File.ReadAllBytes(path), maxLongEdge, frame.Orientation, DecodeTier.Native); + if (preferPreview && ReadPreview(path, frame) is { } preview) return DecodeJpeg(preview, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview); + var request = new DecodeRequest(path, System.Math.Max(0, maxLongEdge), purpose, ApplyOrientation: true, AllowPreview: true); + if (decoder.Decode(request, cancellationToken) is { } decoded) return new DecodedFrame(decoded.Pixels, decoded.Tier); + return ReadPreview(path, frame) is { } fallback ? DecodeJpeg(fallback, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview) : null; + } + + private static DecodedFrame? DecodeJpeg(byte[] data, int maxLongEdge, int orientation, DecodeTier tier) + { + ImageBuffer decoded; + try + { + decoded = JpegDecoder.DecodeToLinear(data, new JpegDecodeOptions { MaxLongEdge = System.Math.Max(0, maxLongEdge) }, expandGray: true); + } + catch (InvalidDataException) + { + return null; + } + if (orientation is <= 1 or > 8) return new DecodedFrame(decoded, tier); + using (decoded) return new DecodedFrame(PixelConvert.ApplyOrientation(decoded, orientation), tier); + } + + private static byte[]? ReadPreview(string path, SidecarFrame frame) + { + if (frame.PreviewOffset is not { } offset || frame.PreviewLength is not ({ } length and > 0 and < int.MaxValue)) return null; + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 4096, FileOptions.RandomAccess); + if (offset < 0 || offset + length > stream.Length) return null; + var data = new byte[length]; + stream.Position = offset; + stream.ReadExactly(data); + return data; + } + + private sealed record DecodedFrame(ImageBuffer Pixels, DecodeTier Tier) : IDisposable + { + public void Dispose() => Pixels.Dispose(); + } +} diff --git a/src/Titano.Core/Archiving/FrameVerifier.cs b/src/Titano.Core/Archiving/FrameVerifier.cs new file mode 100644 index 00000000..553a8051 --- /dev/null +++ b/src/Titano.Core/Archiving/FrameVerifier.cs @@ -0,0 +1,54 @@ +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +public enum FrameHealth +{ + /// Same size and SHA-256 as recorded. + Ok, + + /// No hash was recorded (adopted file); it has been computed now. + Hashed, + + Missing, + + /// A different size: the file was replaced or edited. + Changed, + + /// Same size, different content: bit rot or a damaged copy. + Corrupt, +} + +/// The hash computed now (null when the file is missing or changed size). +public sealed record FrameCheck(SidecarFrame Frame, FrameHealth Health, string? Sha256) +{ + /// Frame flags after this check: / set or cleared. + public FrameFlags Flags => Health switch + { + FrameHealth.Missing => (Frame.Flags & ~FrameFlags.Corrupt) | FrameFlags.Missing, + FrameHealth.Changed or FrameHealth.Corrupt => (Frame.Flags & ~FrameFlags.Missing) | FrameFlags.Corrupt, + _ => Frame.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt), + }; + + /// The sidecar record after this check (flags, and the hash of a newly hashed file). + public SidecarFrame Updated => Frame with { Flags = Flags, Sha256 = Health == FrameHealth.Hashed ? Sha256 : Frame.Sha256 }; + + public bool ChangesRecord => Updated != Frame; +} + +/// Re-hashes originals against their sidecar records (BRIEF §12.5 Verify). +public static class FrameVerifier +{ + public static async Task CheckAsync(SequencePaths sequence, SidecarFrame frame, Action? bytesRead = null, + CancellationToken cancellationToken = default) + { + var info = new FileInfo(sequence.File(frame.File)); + if (!info.Exists) return new FrameCheck(frame, FrameHealth.Missing, null); + if (info.Length != frame.Size) return new FrameCheck(frame, FrameHealth.Changed, null); + var sha = await ContentHash.Sha256Async(info.FullName, bytesRead, cancellationToken).ConfigureAwait(false); + var health = frame.Sha256 is null ? FrameHealth.Hashed + : sha.Equals(frame.Sha256, StringComparison.OrdinalIgnoreCase) ? FrameHealth.Ok + : FrameHealth.Corrupt; + return new FrameCheck(frame, health, sha); + } +} diff --git a/src/Titano.Core/Archiving/IndexSnapshot.cs b/src/Titano.Core/Archiving/IndexSnapshot.cs new file mode 100644 index 00000000..55b54583 --- /dev/null +++ b/src/Titano.Core/Archiving/IndexSnapshot.cs @@ -0,0 +1,127 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; +using Titano.Core.IO; + +namespace Titano.Core.Archiving; + +/// +/// .titano/index-snapshot.bin (BRIEF §7): every sidecar of the archive in one file, so a rebuild reads one file +/// sequentially instead of thousands of small ones over a share. Optional and never authoritative: an entry is used +/// only when its sidecar still has the recorded size and time, and a file whose trailing SHA-256 does not match is +/// ignored entirely. +/// +/// Layout: "TISN", version (u16), reserved (u16), then per sidecar: folder length (u16), UTF-8 folder, +/// size (i64), modified UTC ticks (i64), length (i32), sidecar bytes; finally the SHA-256 of everything before it. +public sealed class IndexSnapshot +{ + private const ushort Version = 1; + private static ReadOnlySpan Magic => "TISN"u8; + + private readonly Dictionary _entries; + + private IndexSnapshot(Dictionary entries) => _entries = entries; + + public int Count => _entries.Count; + + /// The snapshot at , or null when it is absent, damaged or of another version. + public static IndexSnapshot? TryLoad(string path) + { + byte[] data; + try + { + if (!File.Exists(path)) return null; + data = File.ReadAllBytes(path); + } + catch (IOException) + { + return null; + } + if (data.Length < 8 + 32 || !data.AsSpan(0, 4).SequenceEqual(Magic) || BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan(4)) != Version) + return null; + var body = data.AsSpan(0, data.Length - 32); + if (!SHA256.HashData(body).AsSpan().SequenceEqual(data.AsSpan(data.Length - 32))) return null; + + var entries = new Dictionary(StringComparer.Ordinal); + var position = 8; + while (position < body.Length) + { + if (position + 2 > body.Length) return null; + var folderLength = BinaryPrimitives.ReadUInt16LittleEndian(body[position..]); + position += 2; + if (position + folderLength + 20 > body.Length) return null; + var folder = Encoding.UTF8.GetString(body.Slice(position, folderLength)); + position += folderLength; + var size = BinaryPrimitives.ReadInt64LittleEndian(body[position..]); + var ticks = BinaryPrimitives.ReadInt64LittleEndian(body[(position + 8)..]); + var length = BinaryPrimitives.ReadInt32LittleEndian(body[(position + 16)..]); + position += 20; + if (length < 0 || ticks < 0 || ticks > DateTime.MaxValue.Ticks || position + length > body.Length) return null; + entries[folder] = (new FileStamp(size, new DateTime(ticks, DateTimeKind.Utc)), body.Slice(position, length).ToArray()); + position += length; + } + return new IndexSnapshot(entries); + } + + /// The recorded sidecar bytes of when its current stamp still matches. + public byte[]? TryGet(string folder, FileStamp current) => + _entries.TryGetValue(folder, out var entry) && entry.Stamp == current ? entry.Bytes : null; + + /// Streams a new snapshot to a temporary file; replaces the old one atomically. + public sealed class Writer : IDisposable + { + private readonly string _path; + private readonly string _temp; + private readonly FileStream _stream; + private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + private bool _committed; + + public Writer(string path) + { + _path = path; + _temp = AtomicFile.TempPathFor(path); + _stream = new FileStream(_temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16); + Span header = stackalloc byte[8]; + header.Clear(); + Magic.CopyTo(header); + BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version); + Write(header); + } + + public void Add(string folder, FileStamp stamp, ReadOnlySpan sidecar) + { + var name = Encoding.UTF8.GetBytes(folder); + Span head = stackalloc byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(head, checked((ushort)name.Length)); + Write(head); + Write(name); + Span fields = stackalloc byte[20]; + BinaryPrimitives.WriteInt64LittleEndian(fields, stamp.Size); + BinaryPrimitives.WriteInt64LittleEndian(fields[8..], stamp.ModifiedUtc.Ticks); + BinaryPrimitives.WriteInt32LittleEndian(fields[16..], sidecar.Length); + Write(fields); + Write(sidecar); + } + + public void Commit() + { + _stream.Write(_hash.GetHashAndReset()); + _stream.Dispose(); + AtomicFile.Commit(_temp, _path); + _committed = true; + } + + public void Dispose() + { + _stream.Dispose(); + _hash.Dispose(); + if (!_committed && File.Exists(_temp)) File.Delete(_temp); + } + + private void Write(ReadOnlySpan data) + { + _hash.AppendData(data); + _stream.Write(data); + } + } +} diff --git a/src/Titano.Core/Archiving/IngestJournal.cs b/src/Titano.Core/Archiving/IngestJournal.cs new file mode 100644 index 00000000..ec6baf25 --- /dev/null +++ b/src/Titano.Core/Archiving/IngestJournal.cs @@ -0,0 +1,52 @@ +using System.Text.Json; +using Titano.Core.IO; + +namespace Titano.Core.Archiving; + +/// +/// Write-ahead record of an ingest batch (.titano/ingest.pending.json): the frames whose files are about to be +/// moved into the sequence folder. Written before the moves and removed once sequence.json lists them, so a +/// crash in between is repaired by instead of leaving unindexed originals behind. +/// +public sealed record IngestJournal(IReadOnlyList Frames) +{ + public static void Write(SequencePaths sequence, IngestJournal journal) + { + Directory.CreateDirectory(sequence.MetaDirectory); + AtomicFile.WriteAllBytes(sequence.IngestJournalPath, JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJson.Default.IngestJournal)); + } + + public static void Delete(SequencePaths sequence) + { + if (File.Exists(sequence.IngestJournalPath)) File.Delete(sequence.IngestJournalPath); + } + + /// + /// Completes an interrupted batch: frames whose file reached the sequence folder (same size) but are missing from + /// are added in capture order. Returns the manifest to write, or null when nothing has + /// to change. The journal is left in place; it once the result is written. + /// + public static SequenceManifest? Recover(SequencePaths sequence, SequenceManifest manifest) + { + if (!File.Exists(sequence.IngestJournalPath)) return null; + IngestJournal? journal; + try + { + journal = JsonSerializer.Deserialize(File.ReadAllBytes(sequence.IngestJournalPath), ArchiveJson.Default.IngestJournal); + } + catch (JsonException) + { + journal = null; // torn while being written: nothing was moved yet + } + if (journal is null) return null; + var known = manifest.Frames.Select(f => f.Id).ToHashSet(); + var moved = journal.Frames.Where(f => !known.Contains(f.Id) && FileHasSize(sequence.File(f.File), f.Size)).ToList(); + return moved.Count == 0 ? null : SequenceFrames.Insert(manifest, moved); + } + + private static bool FileHasSize(string path, long size) + { + var info = new FileInfo(path); + return info.Exists && info.Length == size; + } +} diff --git a/src/Titano.Core/Archiving/LeaseFile.cs b/src/Titano.Core/Archiving/LeaseFile.cs new file mode 100644 index 00000000..7a460481 --- /dev/null +++ b/src/Titano.Core/Archiving/LeaseFile.cs @@ -0,0 +1,241 @@ +using System.Text.Json; +using Titano.Core.Diagnostics; +using Titano.Core.Localization; + +namespace Titano.Core.Archiving; + +/// Content of a lease file: who holds it and until when. +public sealed record LeaseRecord(string Token, string Owner, string Host, int ProcessId, DateTimeOffset Acquired, DateTimeOffset Expires); + +/// The lease is held by someone else and did not free up in time. +public sealed class LeaseUnavailableException(string name, LeaseRecord? holder) + : IOException(Loc.L("Archive.Error.Busy", holder?.Owner ?? "?")) +{ + public string Name { get; } = name; + public LeaseRecord? Holder { get; } = holder; +} + +/// +/// Cross-process exclusive lease stored as .titano/locks/<name>.lease (BRIEF §7), for writers that share +/// an archive over a network (server and client). Creation uses create-new semantics, which SMB and local file +/// systems both make atomic. A lease past its expiry is stale (its holder crashed or lost the share) and is taken +/// over by renaming it aside, which only one contender can do. The holder renews it in the background while held. +/// +public sealed class LeaseFile : IDisposable +{ + public static readonly TimeSpan DefaultDuration = TimeSpan.FromMinutes(2); + + private const string Extension = ".lease"; + private const string StaleExtension = ".stale"; + private readonly Lock _gate = new(); + private readonly Timer _renewal; + private LeaseRecord _record; + private bool _released; + + private LeaseFile(string path, string name, LeaseRecord record, TimeSpan duration) + { + Path = path; + Name = name; + _record = record; + Duration = duration; + var period = duration / 3; + _renewal = new Timer(_ => Renew(), null, period, period); + } + + public string Path { get; } + public string Name { get; } + public TimeSpan Duration { get; } + public LeaseRecord Record => _record; + + /// False once a renewal found the lease taken over (it expired while this process was stalled). + public bool IsHeld { get; private set; } = true; + + /// Takes the lease when it is free or stale; null when someone else holds it. + public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null) + { + var length = duration ?? DefaultDuration; + var path = PathFor(locksDirectory, name); + Directory.CreateDirectory(locksDirectory); + for (var attempt = 0; attempt < 3; attempt++) + { + var now = DateTimeOffset.UtcNow; + var record = new LeaseRecord(Guid.NewGuid().ToString("N"), owner, Environment.MachineName, Environment.ProcessId, now, now + length); + if (TryCreate(path, record)) return new LeaseFile(path, name, record, length); + if (!IsStale(path, now)) return null; + BreakStale(path, now); + } + return null; + } + + /// Waits up to for the lease, polling with back-off. + /// Still held by someone else after . + public static async Task AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait, + TimeSpan? duration = null, CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow + wait; + var delay = TimeSpan.FromMilliseconds(20); + while (true) + { + if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease; + if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name))); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, 500)); + } + } + + /// The current holder of a lease, or null when free or unreadable. + public static LeaseRecord? Read(string path) + { + try + { + return JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.LeaseRecord); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + return null; + } + } + + /// Removes every stale lease in (maintenance); returns their names. + public static IReadOnlyList CleanupStale(string locksDirectory, bool dryRun) + { + if (!Directory.Exists(locksDirectory)) return []; + var now = DateTimeOffset.UtcNow; + var removed = new List(); + foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + Extension)) + { + if (!IsStale(path, now)) continue; + removed.Add(System.IO.Path.GetFileNameWithoutExtension(path)); + if (!dryRun) BreakStale(path, now); + } + if (!dryRun) + { + foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + StaleExtension)) + { + if (File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime) TryDelete(path); + } + } + return removed; + } + + /// Extends the expiry; false when the lease is no longer ours. + public bool Renew() + { + lock (_gate) + { + if (_released || !IsHeld) return false; + if (Read(Path)?.Token != _record.Token) + { + IsHeld = false; + Log.Warning("archive", $"Lease {Name} was taken over by another writer"); + return false; + } + var renewed = _record with { Expires = DateTimeOffset.UtcNow + Duration }; + try + { + // In-place rewrite: a rename would open a window in which the lease looks free. + using var fs = new FileStream(Path, FileMode.Truncate, FileAccess.Write, FileShare.Read); + fs.Write(JsonSerializer.SerializeToUtf8Bytes(renewed, ArchiveJson.Default.LeaseRecord)); + fs.Flush(flushToDisk: true); + _record = renewed; + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Runs on a timer thread: an escaping exception would end the process. + Log.Warning("archive", $"Lease {Name} could not be renewed", ex); + return false; + } + } + } + + public void Dispose() + { + _renewal.Dispose(); + lock (_gate) + { + if (_released) return; + _released = true; + if (!IsHeld || Read(Path)?.Token != _record.Token) return; + try + { + File.Delete(Path); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // It expires on its own. + Log.Warning("archive", $"Lease {Name} could not be released", ex); + } + } + } + + internal static string PathFor(string locksDirectory, string name) => System.IO.Path.Combine(locksDirectory, name + Extension); + + private static bool TryCreate(string path, LeaseRecord record) + { + try + { + using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read); + fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord)); + fs.Flush(flushToDisk: true); + return true; + } + catch (IOException) when (File.Exists(path)) + { + return false; + } + } + + /// Expired, or unreadable for longer than a lease lasts (a holder that crashed while creating it). + private static bool IsStale(string path, DateTimeOffset now) + { + if (Read(path) is { } record) return record.Expires < now; + try + { + return File.Exists(path) && File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime; + } + catch (IOException) + { + return false; + } + } + + private static void BreakStale(string path, DateTimeOffset now) + { + var aside = path + "." + Guid.NewGuid().ToString("N")[..8] + StaleExtension; + try + { + File.Move(path, aside); + } + catch (IOException) + { + return; // another contender broke it first + } + if (IsStale(aside, now)) + { + TryDelete(aside); + return; + } + // A contender renewed or re-created the lease between our check and the rename: give it back. + try + { + File.Move(aside, path, overwrite: false); + } + catch (IOException) + { + TryDelete(aside); + } + } + + private static void TryDelete(string path) + { + try + { + File.Delete(path); + } + catch (IOException) + { + // A leftover file is harmless; maintenance removes it. + } + } +} diff --git a/src/Titano.Core/Archiving/ManifestStore.cs b/src/Titano.Core/Archiving/ManifestStore.cs new file mode 100644 index 00000000..62a3a11b --- /dev/null +++ b/src/Titano.Core/Archiving/ManifestStore.cs @@ -0,0 +1,84 @@ +using System.Text.Json; +using Titano.Core.IO; +using Titano.Core.Localization; + +namespace Titano.Core.Archiving; + +/// A sidecar written by a newer Titano: it can be read, but this build refuses to rewrite it. +public sealed class NewerFormatException(string path, int version) + : InvalidOperationException(Loc.L("Archive.Error.NewerFormat", System.IO.Path.GetFileName(path), version)) +{ + public string Path { get; } = path; + public int Version { get; } = version; +} + +/// +/// Reads and writes the JSON files of the layout. Reading tolerates unknown fields and enum values; every write +/// goes through (temp → fsync → rename), so a crash leaves the previous file intact. +/// +public static class ManifestStore +{ + /// The file is not valid JSON or lacks required fields. + public static ArchiveManifest ReadArchive(string path) => Parse(File.ReadAllBytes(path), path, ArchiveJson.Default.ArchiveManifest); + + public static void WriteArchive(string path, ArchiveManifest manifest) + { + RefuseNewer(path, manifest.SchemaVersion, ArchiveManifest.CurrentSchemaVersion); + AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJsonIndented.Default.ArchiveManifest)); + } + + /// The file is not valid JSON or lacks required fields. + public static SequenceManifest ReadSequence(string path) => ParseSequence(File.ReadAllBytes(path), path); + + /// Parses sidecar bytes (from disk or from the index snapshot). + public static SequenceManifest ParseSequence(ReadOnlySpan utf8, string source) => Parse(utf8, source, ArchiveJson.Default.SequenceManifest); + + /// Replaces the sidecar atomically. Compact JSON: a 100k-frame sidecar stays around 35 MB. + public static void WriteSequence(string path, SequenceManifest manifest) + { + RefuseNewer(path, manifest.SchemaVersion, SequenceManifest.CurrentSchemaVersion); + AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest)); + } + + /// + /// Creates the sidecar only if none exists (rename without replace), which makes it the claim on a new + /// sequence folder between processes sharing the archive. False when another sidecar is already there. + /// + public static bool TryCreateSequence(string path, SequenceManifest manifest) + { + if (File.Exists(path)) return false; + var temp = AtomicFile.TempPathFor(path); + File.WriteAllBytes(temp, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest)); + try + { + using (var fs = new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) fs.Flush(flushToDisk: true); + File.Move(temp, path, overwrite: false); + return true; + } + catch (IOException) when (File.Exists(path)) + { + return false; + } + finally + { + if (File.Exists(temp)) File.Delete(temp); + } + } + + private static T Parse(ReadOnlySpan utf8, string source, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo) + { + try + { + return JsonSerializer.Deserialize(utf8, typeInfo) ?? throw new InvalidDataException($"{source} is empty."); + } + catch (JsonException ex) + { + throw new InvalidDataException($"{source} is not a valid Titano file: {ex.Message}", ex); + } + } + + private static void RefuseNewer(string path, int version, int supported) + { + if (version > supported) throw new NewerFormatException(path, version); + } +} diff --git a/src/Titano.Core/Archiving/MassRename.cs b/src/Titano.Core/Archiving/MassRename.cs new file mode 100644 index 00000000..caf423c9 --- /dev/null +++ b/src/Titano.Core/Archiving/MassRename.cs @@ -0,0 +1,131 @@ +using System.Globalization; +using System.Text.Json; +using Titano.Core.IO; +using Titano.Core.Naming; + +namespace Titano.Core.Archiving; + +/// What the planner needs to know about a sequence to rename it. +public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens); + +/// One sequence's change: its folder, its title, or both. +public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle) +{ + public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal); +} + +/// The exact changes a mass rename will make (its dry run); applying it makes exactly these. +public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList Steps); + +/// A plan being applied, with the steps already completed; lets an interrupted rename resume (BRIEF §7). +public sealed record RenameJournal(RenamePlan Plan, IReadOnlyList Completed) +{ + public static string PathFor(ArchivePaths archive, Guid planId) => Path.Combine(archive.JournalDirectory, $"rename-{planId:N}.json"); + + public static void Write(ArchivePaths archive, RenameJournal journal) + { + Directory.CreateDirectory(archive.JournalDirectory); + AtomicFile.WriteAllBytes(PathFor(archive, journal.Plan.Id), JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJsonIndented.Default.RenameJournal)); + } + + /// Journals left by interrupted renames. + public static IReadOnlyList ListPending(ArchivePaths archive) + { + if (!Directory.Exists(archive.JournalDirectory)) return []; + var list = new List(); + foreach (var path in Directory.EnumerateFiles(archive.JournalDirectory, "rename-*.json").Order(StringComparer.Ordinal)) + { + if (JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.RenameJournal) is { } journal) list.Add(journal); + } + return list; + } + + public static void Delete(ArchivePaths archive, Guid planId) => File.Delete(PathFor(archive, planId)); +} + +/// Builds rename plans from folder/title templates (BRIEF §7.1, §12.5). +public static class RenamePlanner +{ + /// + /// Expands (where {title} is the current title) and then + /// (where {title} is the new title) for every subject. A target folder + /// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"…. + /// Null templates leave that part unchanged; subjects without a change produce no step. + /// + /// Every sequence folder of the archive (renamed or not). + public static RenamePlan Plan(Guid archiveId, IReadOnlyList subjects, string? folderTemplate, string? titleTemplate, + IEnumerable occupied, Func existsOnDisk) + { + var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate); + var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate); + var claimed = new HashSet(occupied, StringComparer.OrdinalIgnoreCase); + var steps = new List(); + foreach (var s in subjects) + { + var context = new NamingContext + { + CapturedAt = s.FirstCapture ?? DateTime.Today, + SessionStart = s.FirstCapture ?? DateTime.Today, + CameraMake = s.CameraMake, + CameraModel = s.CameraModel, + Lens = s.Lens, + Title = s.Title, + }; + var newTitle = title is null ? s.Title : Blank(title.ExpandFile(context)) ?? s.Title; + var newFolder = s.Folder; + // The archive root itself (a folder adopted in place) cannot move; only its title changes. + if (folder is not null && s.Folder.Length > 0 && Blank(folder.ExpandFolder(context with { Title = newTitle })) is { } expanded + && !expanded.Equals(s.Folder, StringComparison.Ordinal)) + { + newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk); + claimed.Add(newFolder); + } + if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle)); + } + return new RenamePlan(Guid.CreateVersion7(), archiveId, steps); + } + + /// + /// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted, + /// a case-only change goes through a temporary name (case-insensitive file systems). + /// + /// The target is occupied by something else, or neither folder exists. + public static void MoveFolder(ArchivePaths archive, RenameStep step) + { + if (!step.MovesFolder) return; + var from = archive.FromRelative(step.FromFolder); + var to = archive.FromRelative(step.ToFolder); + var caseOnly = step.FromFolder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase); + if (Directory.Exists(to) && (!caseOnly || HasExactName(to))) + { + if (Directory.Exists(from) && !caseOnly) throw new IOException($"Rename target {step.ToFolder} already exists."); + return; + } + if (!Directory.Exists(from)) throw new DirectoryNotFoundException($"Sequence folder {step.FromFolder} is missing."); + Directory.CreateDirectory(Path.GetDirectoryName(to)!); + if (caseOnly) + { + var temp = from + ".rename-" + Guid.NewGuid().ToString("N")[..8]; + Directory.Move(from, temp); + Directory.Move(temp, to); + } + else + { + Directory.Move(from, to); + } + } + + private static bool HasExactName(string path) => + Directory.EnumerateDirectories(Path.GetDirectoryName(path)!, Path.GetFileName(path)) + .Any(d => Path.GetFileName(d).Equals(Path.GetFileName(path), StringComparison.Ordinal)); + + private static string? Blank(string value) => string.IsNullOrWhiteSpace(value) ? null : value; + + private static string Unique(string folder, HashSet claimed, Func existsOnDisk) + { + var candidate = folder; + for (var n = 2; claimed.Contains(candidate) || existsOnDisk(candidate); n++) + candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})"); + return candidate; + } +} diff --git a/src/Titano.Core/Archiving/SequenceAdopter.cs b/src/Titano.Core/Archiving/SequenceAdopter.cs new file mode 100644 index 00000000..77d38d90 --- /dev/null +++ b/src/Titano.Core/Archiving/SequenceAdopter.cs @@ -0,0 +1,91 @@ +using Titano.Core.Diagnostics; +using Titano.Core.Metadata; +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +/// Files of a sequence folder that the sidecar does not list, and sidecar frames whose file is gone. +public sealed record SequenceOrphans(string Folder, IReadOnlyList UnindexedFiles, IReadOnlyList MissingFiles) +{ + public bool IsEmpty => UnindexedFiles.Count == 0 && MissingFiles.Count == 0; +} + +/// +/// Builds and reconciles sidecars from the photos found in a folder (BRIEF §7, §12.5 "Adopt folder"). Only light +/// metadata is read; originals are never moved, renamed or rewritten. Hashes are left empty for a verify to fill. +/// +public static class SequenceAdopter +{ + /// A new sidecar for a folder without one. A folder outside the layout (no originals/) is titled by its name. + public static SequenceManifest Build(Guid id, SequencePaths sequence, string? createdBy, CancellationToken cancellationToken = default) + { + var (frames, first) = ReadFrames(sequence, ArchiveScanner.ListPhotos(sequence), cancellationToken); + var now = DateTimeOffset.UtcNow; + var inLayout = Directory.Exists(sequence.Originals); + var manifest = new SequenceManifest + { + Id = id, + Title = inLayout ? null : Path.GetFileName(Path.TrimEndingDirectorySeparator(sequence.Folder)), + Camera = first is null ? null : new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, + Lens = first?.LensModel, + CreatedBy = createdBy, + Created = now, + UpdatedAt = now, + }; + return SequenceFrames.Insert(manifest, frames); + } + + /// Unindexed photos and missing frame files of one sequence. + public static SequenceOrphans FindOrphans(SequencePaths sequence, SequenceManifest manifest) + { + var listed = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal); + var unindexed = ArchiveScanner.ListPhotos(sequence).Where(f => !listed.Contains(f)).ToList(); + var missing = manifest.Frames.Where(f => !File.Exists(sequence.File(f.File))).ToList(); + return new SequenceOrphans(sequence.RelativeFolder, unindexed, missing); + } + + /// + /// Brings a sidecar in line with its folder: unindexed photos are added (in capture order), frames whose file is + /// gone get and frames whose file came back lose it. Null when nothing changed. + /// + public static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken = default) + { + var orphans = FindOrphans(sequence, manifest); + var missing = orphans.MissingFiles.Select(f => f.Id).ToHashSet(); + var flagChanges = new Dictionary>(); + foreach (var frame in manifest.Frames) + { + var isMissing = missing.Contains(frame.Id); + if (isMissing == frame.Flags.HasFlag(FrameFlags.Missing)) continue; + flagChanges[frame.Id] = f => f with { Flags = isMissing ? f.Flags | FrameFlags.Missing : f.Flags & ~FrameFlags.Missing }; + } + if (flagChanges.Count == 0 && orphans.UnindexedFiles.Count == 0) return null; + var updated = flagChanges.Count == 0 ? manifest : SequenceFrames.Update(manifest, flagChanges); + var (added, first) = ReadFrames(sequence, orphans.UnindexedFiles, cancellationToken); + if (updated.Camera is null && first is not null) + updated = updated with { Camera = new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, Lens = updated.Lens ?? first.LensModel }; + return SequenceFrames.Insert(updated, added); + } + + private static (List Frames, FrameMetadata? First) ReadFrames(SequencePaths sequence, IReadOnlyList files, CancellationToken cancellationToken) + { + var frames = new List(files.Count); + FrameMetadata? first = null; + foreach (var file in files) + { + cancellationToken.ThrowIfCancellationRequested(); + var path = sequence.File(file); + try + { + var metadata = MetadataReader.Read(path, cancellationToken); + if (first is null || (metadata.CapturedAt < first.CapturedAt && metadata.Make is not null)) first = metadata; + frames.Add(SequenceFrames.FromMetadata(Guid.CreateVersion7(), file, Path.GetFileName(file), new FileInfo(path).Length, null, metadata)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"Skipped unreadable photo {sequence.ArchiveRelative(file)}", ex); + } + } + return (frames, first); + } +} diff --git a/src/Titano.Core/Archiving/SequenceFrames.cs b/src/Titano.Core/Archiving/SequenceFrames.cs new file mode 100644 index 00000000..4735e3ae --- /dev/null +++ b/src/Titano.Core/Archiving/SequenceFrames.cs @@ -0,0 +1,86 @@ +using Titano.Core.Metadata; +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +/// Edits of the frame list of a ; frames stay in capture order with gap-free indexes. +public static class SequenceFrames +{ + /// Capture time (unknown last), then file name. + public static readonly IComparer CaptureOrder = Comparer.Create((a, b) => + { + var byTime = (a.CapturedAt ?? DateTime.MaxValue).CompareTo(b.CapturedAt ?? DateTime.MaxValue); + return byTime != 0 ? byTime : string.CompareOrdinal(a.File, b.File); + }); + + /// A frame record from light metadata. + /// Path relative to the sequence folder. + /// Used when the file records no capture time of its own (only a file-system date). + public static SidecarFrame FromMetadata(Guid id, string file, string? original, long size, string? sha256, FrameMetadata metadata, + DateTime? fallbackCapture = null) + { + var useFallback = fallbackCapture is not null && metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown; + return new SidecarFrame + { + Id = id, + File = file, + Original = original, + Size = size, + Sha256 = sha256?.ToLowerInvariant(), + CapturedAt = useFallback ? DateTime.SpecifyKind(fallbackCapture!.Value, DateTimeKind.Unspecified) : metadata.CapturedAt, + UtcOffset = useFallback ? null : metadata.UtcOffsetMinutes, + TimeSource = useFallback ? TimestampSource.Unknown : metadata.TimestampSource, + Exposure = metadata.ExposureTime, + FNumber = metadata.FNumber, + Iso = metadata.Iso, + Focal = metadata.FocalLength, + Bias = metadata.ExposureBias, + Wb = metadata.WhiteBalanceMode, + Orientation = metadata.Orientation, + W = metadata.Width, + H = metadata.Height, + Format = metadata.Format, + PreviewOffset = metadata.LargestPreview?.Offset, + PreviewLength = metadata.LargestPreview?.Length, + }; + } + + /// + /// Merges into the frame list by capture time (existing frames keep their relative order), + /// renumbers indexes and bumps the frames version. + /// + public static SequenceManifest Insert(SequenceManifest manifest, IReadOnlyList added) + { + if (added.Count == 0) return manifest; + var incoming = added.Order(CaptureOrder).ToList(); + var merged = new List(manifest.Frames.Count + incoming.Count); + var next = 0; + foreach (var frame in manifest.Frames) + { + while (next < incoming.Count && CaptureOrder.Compare(incoming[next], frame) < 0) merged.Add(incoming[next++]); + merged.Add(frame); + } + while (next < incoming.Count) merged.Add(incoming[next++]); + return WithFrames(manifest, merged); + } + + /// Replaces the frame list (indexes renumbered) and bumps the frames version. + public static SequenceManifest WithFrames(SequenceManifest manifest, IReadOnlyList frames) + { + var renumbered = new SidecarFrame[frames.Count]; + for (var i = 0; i < frames.Count; i++) renumbered[i] = frames[i].I == i ? frames[i] : frames[i] with { I = i }; + return manifest with + { + Frames = renumbered, + Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + } + + /// Applies per-frame changes by id (flags, hashes); frames not in are kept. + public static SequenceManifest Update(SequenceManifest manifest, IReadOnlyDictionary> changes) + { + var frames = manifest.Frames.Select(f => changes.TryGetValue(f.Id, out var change) ? change(f) : f).ToList(); + return manifest with { Frames = frames, UpdatedAt = DateTimeOffset.UtcNow }; + } +} diff --git a/src/Titano.Core/Archiving/SequenceManifest.cs b/src/Titano.Core/Archiving/SequenceManifest.cs new file mode 100644 index 00000000..dbab9390 --- /dev/null +++ b/src/Titano.Core/Archiving/SequenceManifest.cs @@ -0,0 +1,117 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Imaging; +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +/// +/// .titano/sequence.json, the canonical record of one sequence (BRIEF §7). The database is rebuilt from it, +/// so it carries the ids of the sequence and of every frame. Fields this build does not know are kept in +/// and written back unchanged. +/// +public sealed record SequenceManifest +{ + public const int CurrentSchemaVersion = 1; + + public int SchemaVersion { get; init; } = CurrentSchemaVersion; + public required Guid Id { get; init; } + public string? Title { get; init; } + public SidecarCamera? Camera { get; init; } + public string? Lens { get; init; } + + /// Frames in capture order; equals the position. + public IReadOnlyList Frames { get; init; } = []; + + public IReadOnlyList Tags { get; init; } = []; + public int Rating { get; init; } + public string? Notes { get; init; } + + /// bits, stored as a number so bits of newer builds survive. + public SequenceStatus Status { get; init; } + + public ManifestVersions Versions { get; init; } = new(); + + /// User id or client name that created the sequence. + public string? CreatedBy { get; init; } + + public DateTimeOffset Created { get; init; } + public DateTimeOffset UpdatedAt { get; init; } + public DateTimeOffset? VerifiedAt { get; init; } + + /// Session keys of automatic grouping (uploads, card imports) that resolve to this sequence. + public IReadOnlyList SessionKeys { get; init; } = []; + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +public sealed record SidecarCamera +{ + public string? Make { get; init; } + public string? Model { get; init; } + public string? Serial { get; init; } + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// Counters the sync protocol compares (BRIEF §14.5). +public sealed record ManifestVersions +{ + public int Settings { get; init; } + public int Frames { get; init; } + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// One frame of ; property names follow BRIEF §7. +public sealed record SidecarFrame +{ + public required Guid Id { get; init; } + + /// Index in capture order, from 0. + public int I { get; init; } + + /// Path relative to the sequence folder, forward slashes (originals/… for imported frames). + public required string File { get; init; } + + /// Name the file had on the card or upload source. + public string? Original { get; init; } + + public long Size { get; init; } + + /// Lower-case hex SHA-256; null until a verify hashes an adopted file. + public string? Sha256 { get; init; } + + /// Camera wall-clock time with sub-seconds. + public DateTime? CapturedAt { get; init; } + + public int? UtcOffset { get; init; } + + [JsonConverter(typeof(TolerantEnumConverter))] + public TimestampSource TimeSource { get; init; } + + public double? Exposure { get; init; } + public double? FNumber { get; init; } + public int? Iso { get; init; } + public double? Focal { get; init; } + public double? Bias { get; init; } + public int? Wb { get; init; } + public int Orientation { get; init; } = 1; + public int W { get; init; } + public int H { get; init; } + + [JsonConverter(typeof(TolerantEnumConverter))] + public ImageFileFormat Format { get; init; } + + public long? PreviewOffset { get; init; } + public long? PreviewLength { get; init; } + + /// bits, stored as a number so bits of newer builds survive. + public FrameFlags Flags { get; init; } + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} diff --git a/src/Titano.Core/Archiving/ThumbPack.cs b/src/Titano.Core/Archiving/ThumbPack.cs new file mode 100644 index 00000000..48bf29df --- /dev/null +++ b/src/Titano.Core/Archiving/ThumbPack.cs @@ -0,0 +1,389 @@ +using System.Buffers.Binary; +using Titano.Core.Hashing; + +namespace Titano.Core.Archiving; + +/// One thumbnail of a pack: where its JPEG lies and which source it was made from. +/// 32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale. +public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc); + +/// +/// thumbs.pack: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence +/// instead of thousands. Little-endian blocks after a 16-byte header: +/// +/// TENT entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes. +/// TIDX index: count, one row per frame (latest entry wins), CRC-32 of the rows. +/// TFTR footer (last 16 bytes): offset of the index block and its CRC. +/// +/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index +/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall +/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it). +/// +public static class ThumbPack +{ + public const int HashLength = 32; + internal const int HeaderLength = 16; + internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4; + internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4; + internal const int FooterLength = 16; + internal const ushort Version = 1; + + /// Largest thumbnail accepted; anything bigger in a header is corruption. + internal const int MaxEntryLength = 16 << 20; + + internal static ReadOnlySpan Magic => "TTPK"u8; + internal static ReadOnlySpan EntryTag => "TENT"u8; + internal static ReadOnlySpan IndexTag => "TIDX"u8; + internal static ReadOnlySpan FooterTag => "TFTR"u8; + + internal static void WriteHeader(Stream stream) + { + Span header = stackalloc byte[HeaderLength]; + header.Clear(); + Magic.CopyTo(header); + BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version); + stream.Write(header); + } + + internal static bool HasValidHeader(Stream stream) + { + if (stream.Length < HeaderLength) return false; + Span header = stackalloc byte[HeaderLength]; + stream.Position = 0; + stream.ReadExactly(header); + return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version; + } + + /// + /// The latest entry per frame and where the valid content ends: after the footer when the index is intact + /// ( = start of that index), otherwise after the last complete block. + /// + internal static Dictionary Load(Stream stream, out long validEnd, out long? indexOffset) + { + indexOffset = null; + if (TryLoadIndex(stream, out var entries, out var offset)) + { + validEnd = stream.Length; + indexOffset = offset; + return entries; + } + return Scan(stream, out validEnd); + } + + private static bool TryLoadIndex(Stream stream, out Dictionary entries, out long indexOffset) + { + entries = []; + indexOffset = 0; + var length = stream.Length; + if (length < HeaderLength + 12 + FooterLength) return false; + Span footer = stackalloc byte[FooterLength]; + stream.Position = length - FooterLength; + stream.ReadExactly(footer); + if (!footer[..4].SequenceEqual(FooterTag)) return false; + indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]); + if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false; + if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false; + + Span head = stackalloc byte[8]; + stream.Position = indexOffset; + stream.ReadExactly(head); + var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]); + if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false; + var rows = new byte[count * IndexRowLength + 4]; + stream.ReadExactly(rows); + if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false; + for (var i = 0; i < count; i++) + { + var row = rows.AsSpan(i * IndexRowLength, IndexRowLength); + var entry = new ThumbEntry( + BinaryPrimitives.ReadInt32LittleEndian(row), + BinaryPrimitives.ReadInt64LittleEndian(row[4..]), + BinaryPrimitives.ReadInt32LittleEndian(row[12..]), + BinaryPrimitives.ReadUInt16LittleEndian(row[16..]), + BinaryPrimitives.ReadUInt16LittleEndian(row[18..]), + row.Slice(20, HashLength).ToArray(), + BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..])); + if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false; + entries[entry.FrameIndex] = entry; + } + return true; + } + + /// Walks block headers from the start; stops at the first incomplete or unknown block. + private static Dictionary Scan(Stream stream, out long validEnd) + { + var entries = new Dictionary(); + var length = stream.Length; + var position = (long)HeaderLength; + var header = new byte[EntryHeaderLength]; + while (position + 4 <= length) + { + stream.Position = position; + stream.ReadExactly(header.AsSpan(0, 4)); + var tag = header.AsSpan(0, 4); + long next; + if (tag.SequenceEqual(EntryTag)) + { + if (position + EntryHeaderLength > length) break; + stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4)); + var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength)); + if (size is < 0 or > MaxEntryLength) break; + next = position + EntryHeaderLength + size; + if (next > length) break; + var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4)); + entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size, + BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)), + header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength))); + } + else if (tag.SequenceEqual(IndexTag)) + { + if (position + 8 > length) break; + stream.ReadExactly(header.AsSpan(4, 4)); + var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4)); + if (count < 0) break; + next = position + 8 + (long)count * IndexRowLength + 4; + if (next > length) break; + } + else if (tag.SequenceEqual(FooterTag)) + { + next = position + FooterLength; + if (next > length) break; + } + else + { + break; + } + position = next; + } + validEnd = position; + return entries; + } + + /// Writes one entry block at the stream position and returns its entry. + internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan jpeg) + { + var crc = Crc32.Compute(jpeg); + Span header = stackalloc byte[EntryHeaderLength]; + EntryTag.CopyTo(header); + BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex); + BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width); + BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height); + sourceHash.CopyTo(header[12..]); + BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length); + BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc); + var payload = stream.Position + EntryHeaderLength; + stream.Write(header); + stream.Write(jpeg); + return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc); + } + + internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection entries) + { + var indexOffset = stream.Position; + var block = new byte[8 + entries.Count * IndexRowLength + 4]; + IndexTag.CopyTo(block); + BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count); + var i = 0; + foreach (var e in entries.OrderBy(e => e.FrameIndex)) + { + var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength); + BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex); + BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset); + BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length); + BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width); + BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height); + e.SourceHash.CopyTo(row[20..]); + BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc); + } + var rowsLength = entries.Count * IndexRowLength; + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength))); + stream.Write(block); + + Span footer = stackalloc byte[FooterLength]; + FooterTag.CopyTo(footer); + BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset); + BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12])); + stream.Write(footer); + } + + /// Size of a pack holding only ; the rest of a file is reclaimable garbage. + public static long LiveBytes(IEnumerable entries) + { + long count = 0, payload = 0; + foreach (var e in entries) + { + count++; + payload += EntryHeaderLength + e.Length; + } + return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength; + } + + /// A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known. + public static byte[] Fingerprint(string? sha256, string file, long size) => + sha256 is { Length: HashLength * 2 } + ? Convert.FromHexString(sha256) + : System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}")); +} + +/// Random access to the thumbnails of a pack without loading the file; safe while a writer appends. +public sealed class ThumbPackReader : IDisposable +{ + private readonly FileStream? _stream; + private readonly Lock _gate = new(); + + private ThumbPackReader(FileStream? stream, Dictionary entries) + { + _stream = stream; + Entries = entries; + } + + /// The latest entry of every frame that has one. + public IReadOnlyDictionary Entries { get; } + + /// Bytes held by replaced entries (reclaimed by ). + public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values)); + + public long Length => _stream?.Length ?? 0; + + /// Opens a pack; a missing or damaged pack reads as empty. + public static ThumbPackReader Open(string path) + { + if (!File.Exists(path)) return new ThumbPackReader(null, []); + var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess); + try + { + var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : []; + return new ThumbPackReader(stream, entries); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// The JPEG of a frame, or null when absent or damaged (CRC mismatch). + public byte[]? Read(int frameIndex) + { + if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null; + var data = new byte[entry.Length]; + lock (_gate) + { + if (entry.Offset + entry.Length > _stream.Length) return null; + _stream.Position = entry.Offset; + _stream.ReadExactly(data); + } + return Crc32.Compute(data) == entry.Crc ? data : null; + } + + public void Dispose() => _stream?.Dispose(); +} + +/// Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it). +public sealed class ThumbPackWriter : IDisposable +{ + private readonly FileStream _stream; + private readonly Dictionary _entries; + private long? _indexOffset; + private bool _dirty; + + private ThumbPackWriter(FileStream stream, Dictionary entries, long? indexOffset) + { + _stream = stream; + _entries = entries; + _indexOffset = indexOffset; + } + + public IReadOnlyDictionary Entries => _entries; + + /// Opens or creates a pack, dropping an incomplete tail left by a crash. + public static ThumbPackWriter Open(string path) + { + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!); + var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16); + try + { + if (!ThumbPack.HasValidHeader(stream)) + { + // Not a pack (or a header torn by a crash): thumbnails are regenerable, start over. + stream.SetLength(0); + ThumbPack.WriteHeader(stream); + return new ThumbPackWriter(stream, [], null); + } + var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset); + if (validEnd < stream.Length) stream.SetLength(validEnd); + return new ThumbPackWriter(stream, entries, indexOffset); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// Appends a thumbnail; it replaces any earlier entry of the frame once committed. + public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan jpeg) + { + ArgumentOutOfRangeException.ThrowIfNegative(frameIndex); + if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash)); + if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg)); + if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width)); + // The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk). + _stream.Position = _indexOffset ?? _stream.Length; + if (_indexOffset is { } offset) _stream.SetLength(offset); + _indexOffset = null; + + _entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg); + _dirty = true; + } + + /// Writes the index and footer and flushes to disk. + public void Commit() + { + if (!_dirty) return; + _stream.Position = _stream.Length; + var indexOffset = _stream.Position; + ThumbPack.WriteIndexAndFooter(_stream, _entries.Values); + _stream.Flush(flushToDisk: true); + _indexOffset = indexOffset; + _dirty = false; + } + + /// Rewrites the pack with only its live entries (atomically, so readers keep a consistent file). + public static void Compact(string path) + { + var temp = IO.AtomicFile.TempPathFor(path); + try + { + using (var source = ThumbPackReader.Open(path)) + using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16)) + { + ThumbPack.WriteHeader(target); + var entries = new List(source.Entries.Count); + foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex)) + { + if (source.Read(entry.FrameIndex) is { } jpeg) + entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg)); + } + ThumbPack.WriteIndexAndFooter(target, entries); + } + IO.AtomicFile.Commit(temp, path); + } + finally + { + if (File.Exists(temp)) File.Delete(temp); + } + } + + public void Dispose() + { + try + { + Commit(); + } + finally + { + _stream.Dispose(); + } + } +} diff --git a/src/Titano.Core/Import/CardEraser.cs b/src/Titano.Core/Import/CardEraser.cs new file mode 100644 index 00000000..874d6ea2 --- /dev/null +++ b/src/Titano.Core/Import/CardEraser.cs @@ -0,0 +1,90 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; + +namespace Titano.Core.Import; + +/// A card file that was imported, with the hash recorded at import and where its copy lives. +/// Path below the card root, forward slashes. +/// Absolute path of the archive (or folder) copy. +public sealed record EraseCandidate(string SourcePath, long Size, string Sha256, string CopyPath); + +public enum EraseVerdict +{ + /// Card file and copy both hash to the recorded SHA-256: safe to delete from the card. + Delete, + + SourceMissing, + + /// The card file no longer matches what was imported. + SourceChanged, + + CopyMissing, + + /// The copy no longer matches: the card file is the only good one left. + CopyMismatch, +} + +/// Card file time when checked; execution refuses a file touched since. +public sealed record EraseItem(EraseCandidate Candidate, EraseVerdict Verdict, DateTime? SourceModifiedUtc); + +/// The dry run of "delete imported" (BRIEF §12.4, §13): exactly the files that would be deleted, and why +/// the others are kept. +public sealed record ErasePlan(string SourceRoot, IReadOnlyList Items) +{ + public IEnumerable ToDelete => Items.Where(i => i.Verdict == EraseVerdict.Delete); +} + +public sealed record EraseResult(int Deleted, IReadOnlyList Kept); + +/// +/// Deletes imported photos from a card only after re-hashing both the card file and its copy against the hash +/// recorded at import (BRIEF §0.2 rule 8). Never formats (that is the OS dialog) and never touches folders. +/// +public static class CardEraser +{ + public static async Task PlanAsync(string sourceRoot, IReadOnlyList candidates, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + var items = new List(candidates.Count); + for (var i = 0; i < candidates.Count; i++) + { + var c = candidates[i]; + var source = new FileInfo(Path.Combine(sourceRoot, c.SourcePath)); + var verdict = !source.Exists ? EraseVerdict.SourceMissing + : !File.Exists(c.CopyPath) ? EraseVerdict.CopyMissing + : !await MatchesAsync(c.CopyPath, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.CopyMismatch + : !await MatchesAsync(source.FullName, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.SourceChanged + : EraseVerdict.Delete; + items.Add(new EraseItem(c, verdict, source.Exists ? source.LastWriteTimeUtc : null)); + progress?.Report(i + 1); + } + return new ErasePlan(Path.GetFullPath(sourceRoot), items); + } + + /// Deletes the plan's files whose size and time are unchanged since planning. + public static EraseResult Execute(ErasePlan plan, CancellationToken cancellationToken = default) + { + var deleted = 0; + var kept = plan.Items.Where(i => i.Verdict != EraseVerdict.Delete).ToList(); + foreach (var item in plan.ToDelete) + { + cancellationToken.ThrowIfCancellationRequested(); + var source = new FileInfo(Path.Combine(plan.SourceRoot, item.Candidate.SourcePath)); + var copy = new FileInfo(item.Candidate.CopyPath); + if (!source.Exists || source.Length != item.Candidate.Size || source.LastWriteTimeUtc != item.SourceModifiedUtc + || !copy.Exists || copy.Length != item.Candidate.Size) + { + kept.Add(item with { Verdict = EraseVerdict.SourceChanged }); + continue; + } + source.Delete(); + deleted++; + Log.Info("import", $"Deleted verified {item.Candidate.SourcePath} from the card"); + } + return new EraseResult(deleted, kept); + } + + private static async Task MatchesAsync(string path, EraseCandidate candidate, CancellationToken cancellationToken) => + new FileInfo(path).Length == candidate.Size + && string.Equals(await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false), candidate.Sha256, StringComparison.OrdinalIgnoreCase); +} diff --git a/src/Titano.Core/Import/CardScanner.cs b/src/Titano.Core/Import/CardScanner.cs new file mode 100644 index 00000000..603f5958 --- /dev/null +++ b/src/Titano.Core/Import/CardScanner.cs @@ -0,0 +1,87 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Metadata; + +namespace Titano.Core.Import; + +/// A photo file on a card or source folder. +/// Path below the source root, forward slashes (the card import history key). +public sealed record CardFile(string RelativePath, long Size, DateTime ModifiedUtc); + +/// A card photo with its light metadata. +public sealed record CardPhoto(CardFile File, FrameMetadata Metadata) +{ + /// Capture time, or the file time when the photo records none. + public DateTime CaptureTime => Metadata.CapturedAt ?? File.ModifiedUtc; +} + +/// A file that could not be read while planning; it is listed, never a reason to stop. +public sealed record ImportProblem(string RelativePath, string Message); + +/// Finds photos on a card (BRIEF §12.4, §13): the DCIM tree when present, otherwise the root. +public static class CardScanner +{ + private const int MaxDepth = 12; + + private static readonly EnumerationOptions Listing = new() + { + IgnoreInaccessible = true, + AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System | FileAttributes.Hidden, + RecurseSubdirectories = false, + MatchCasing = MatchCasing.CaseInsensitive, + }; + + /// + /// Photos below : everything under DCIM (camera folders) when the card has one, + /// otherwise the root's photos and, with , its subfolders'. Ordered by path. + /// + public static IReadOnlyList Find(string root, bool includeSubfolders, CancellationToken cancellationToken = default) + { + var dcim = Directory.EnumerateDirectories(root, "DCIM", Listing).FirstOrDefault(); + var start = dcim ?? root; + var recurse = dcim is not null || includeSubfolders; + var files = new List(); + var pending = new Stack<(string Path, int Depth)>(); + pending.Push((start, 0)); + while (pending.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var (directory, depth) = pending.Pop(); + foreach (var file in new DirectoryInfo(directory).EnumerateFiles("*", Listing)) + { + if (ArchiveScanner.IsPhoto(file.Name)) + files.Add(new CardFile(Path.GetRelativePath(root, file.FullName).Replace('\\', '/'), file.Length, file.LastWriteTimeUtc)); + } + if (!recurse || depth >= MaxDepth) continue; + foreach (var child in Directory.EnumerateDirectories(directory, "*", Listing)) + { + if (!Path.GetFileName(child).StartsWith('.')) pending.Push((child, depth + 1)); + } + } + files.Sort((a, b) => string.CompareOrdinal(a.RelativePath, b.RelativePath)); + return files; + } + + /// Reads light metadata of every file; unreadable files become . + public static IReadOnlyList ReadMetadata(string root, IReadOnlyList files, List problems, + IProgress? progress = null, CancellationToken cancellationToken = default) + { + var photos = new List(files.Count); + for (var i = 0; i < files.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var file = files[i]; + try + { + photos.Add(new CardPhoto(file, MetadataReader.Read(Path.Combine(root, file.RelativePath), cancellationToken))); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("import", $"Cannot read {file.RelativePath}", ex); + problems.Add(new ImportProblem(file.RelativePath, ex.Message)); + } + progress?.Report(i + 1); + } + return photos; + } +} diff --git a/src/Titano.Core/Import/FolderImportSink.cs b/src/Titano.Core/Import/FolderImportSink.cs new file mode 100644 index 00000000..8234c666 --- /dev/null +++ b/src/Titano.Core/Import/FolderImportSink.cs @@ -0,0 +1,42 @@ +using Titano.Core.Archiving; + +namespace Titano.Core.Import; + +/// +/// Import destination "Folder" (BRIEF §12.4): photos go to root/<folder>/<name> without sidecars or index. +/// A name already taken by identical content is a duplicate; by other content, the photo gets a suffix. Nothing at the +/// destination is ever replaced. +/// +public sealed class FolderImportSink(string root) : IImportSink +{ + public string StagingDirectory { get; } = Path.Combine(Path.GetFullPath(root), ArchiveLayout.MetaFolder, ArchiveLayout.StagingFolder); + + public async Task> CommitAsync(PlannedSession session, IReadOnlyList files, CancellationToken cancellationToken) + { + var folder = Path.Combine(Path.GetFullPath(root), session.Folder.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(folder); + var results = new List(files.Count); + foreach (var file in files) + { + var name = file.Item.FileName; + var stem = Path.GetFileNameWithoutExtension(name); + var duplicate = false; + for (var n = 2; File.Exists(Path.Combine(folder, name)); n++) + { + if (await IsSameAsync(Path.Combine(folder, name), file, cancellationToken).ConfigureAwait(false)) + { + duplicate = true; + break; + } + name = $"{stem}_{n}{Path.GetExtension(file.Item.FileName)}"; + } + if (!duplicate) File.Move(file.StagedPath, Path.Combine(folder, name), overwrite: false); + results.Add(new SinkCommit(null, session.Folder + "/" + name, duplicate)); + } + return results; + } + + private static async Task IsSameAsync(string existing, StagedFile file, CancellationToken cancellationToken) => + new FileInfo(existing).Length == file.Item.Size + && await ContentHash.Sha256Async(existing, null, cancellationToken).ConfigureAwait(false) == file.Sha256; +} diff --git a/src/Titano.Core/Import/ImportExecutor.cs b/src/Titano.Core/Import/ImportExecutor.cs new file mode 100644 index 00000000..3a8c0728 --- /dev/null +++ b/src/Titano.Core/Import/ImportExecutor.cs @@ -0,0 +1,148 @@ +using Titano.Core.Diagnostics; +using Titano.Core.Localization; + +namespace Titano.Core.Import; + +/// The card or source folder disappeared mid-import (card removed, share lost). The import can resume. +public sealed class ImportSourceUnavailableException(string root, Exception? inner = null) + : IOException(Loc.L("Archive.Error.SourceGone", root), inner) +{ + public string Root { get; } = root; +} + +/// A verified copy waiting in the sink's staging folder. +public sealed record StagedFile(PlannedItem Item, string StagedPath, string Sha256); + +/// Where a staged file ended up. +/// The archive frame (null for a plain folder destination). +/// The destination already held this content; the staged copy was not used. +public sealed record SinkCommit(Guid? FrameId, string RelativePath, bool Duplicate); + +/// Destination of an import: the archive, a plain folder, or a server (BRIEF §12.4). +public interface IImportSink +{ + /// Folder for verified copies, on the destination volume so commits are renames. + string StagingDirectory { get; } + + /// Moves staged files of one session into place; returns one result per file, in order. Files it does + /// not move stay where they are and the executor deletes them. + Task> CommitAsync(PlannedSession session, IReadOnlyList files, CancellationToken cancellationToken); +} + +public enum ImportItemStatus +{ + Copied, + + /// Not copied: imported before, or already in the archive. + Skipped, + + /// Copied and verified, but the destination already held the same content. + Duplicate, + + Failed, +} + +public sealed record ImportItemResult(PlannedSession Session, PlannedItem Item, ImportItemStatus Status, string? Sha256, SinkCommit? Commit, string? Error); + +/// Live figures for the job bar: items and bytes (throughput and ETA come from bytes). +public sealed record ImportProgress(int ItemsDone, int ItemsTotal, long BytesDone, long BytesTotal, int Copied, int Skipped, int Failed, string? Current); + +/// Results committed since the previous batch; is the resume point once they are recorded. +public sealed record ImportBatch(IReadOnlyList Results, int NextItem, ImportProgress Progress); + +/// +/// Executes an (BRIEF §12.4, §13): copies each photo to staging with hashing and read-back +/// verification, commits verified copies to the sink in batches, reports per-file errors without stopping, and can +/// resume from any batch boundary. When the source disappears it commits what is already verified and throws +/// . +/// +public sealed class ImportExecutor(ImportPlan plan, IImportSink sink) +{ + private const int BatchSize = 32; + + /// Number of plan items already done (the last recorded). + /// Called after each batch; record the results, then checkpoint . + /// Called before and after each copy (throughput and ETA come from the bytes). + public async Task RunAsync(int startAt, bool verify, Func committed, + Func? progress = null, CancellationToken cancellationToken = default) + { + var items = plan.Sessions.SelectMany(s => s.Items.Select(i => (Session: s, Item: i))).ToList(); + var totalBytes = items.Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size); + var bytesDone = items.Take(startAt).Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size); + int copied = 0, skipped = 0, failed = 0; + var results = new List(); + var staged = new List(); + PlannedSession? stagedSession = null; + var next = startAt; + + ImportProgress Snapshot(string? current) => new(next, items.Count, bytesDone, totalBytes, copied, skipped, failed, current); + + async Task FlushAsync() + { + if (staged.Count > 0) + { + var commits = await sink.CommitAsync(stagedSession!, staged, cancellationToken).ConfigureAwait(false); + for (var i = 0; i < staged.Count; i++) + { + var duplicate = commits[i].Duplicate; + results.Add(new ImportItemResult(stagedSession!, staged[i].Item, duplicate ? ImportItemStatus.Duplicate : ImportItemStatus.Copied, + staged[i].Sha256, commits[i], null)); + if (duplicate) skipped++; + else copied++; + DeleteLeftover(staged[i].StagedPath); + } + staged.Clear(); + stagedSession = null; + } + if (results.Count == 0) return; + await committed(new ImportBatch([.. results], next, Snapshot(null))).ConfigureAwait(false); + results.Clear(); + } + + Directory.CreateDirectory(sink.StagingDirectory); + for (; next < items.Count; next++) + { + cancellationToken.ThrowIfCancellationRequested(); + var (session, item) = items[next]; + if ((stagedSession is not null && !ReferenceEquals(stagedSession, session)) || staged.Count + results.Count >= BatchSize) + await FlushAsync().ConfigureAwait(false); + if (item.Action != ImportAction.Copy) + { + skipped++; + results.Add(new ImportItemResult(session, item, ImportItemStatus.Skipped, item.Sha256, null, null)); + continue; + } + var source = Path.Combine(plan.SourceRoot, item.SourcePath); + if (progress is not null) await progress(Snapshot(item.SourcePath)).ConfigureAwait(false); + var target = Path.Combine(sink.StagingDirectory, $"{plan.Id:N}-{next:D6}{Path.GetExtension(item.FileName)}"); + DeleteLeftover(target); + try + { + if (new FileInfo(source) is not { Exists: true } info || info.Length != item.Size) + throw new IOException(Loc.L("Archive.Error.SourceChanged", item.SourcePath)); + var sha = await VerifiedCopy.CopyAsync(source, target, verify, read => bytesDone += read, cancellationToken).ConfigureAwait(false); + staged.Add(new StagedFile(item, target, sha)); + stagedSession = session; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + if (!Directory.Exists(plan.SourceRoot)) + { + await FlushAsync().ConfigureAwait(false); + throw new ImportSourceUnavailableException(plan.SourceRoot, ex); + } + Log.Warning("import", $"Copy of {item.SourcePath} failed", ex); + failed++; + results.Add(new ImportItemResult(session, item, ImportItemStatus.Failed, null, null, ex.Message)); + } + } + await FlushAsync().ConfigureAwait(false); + return Snapshot(null); + } + + private static void DeleteLeftover(string staged) + { + // Staged files are this import's own copies of card files; the originals stay on the card. + if (File.Exists(staged)) File.Delete(staged); + } +} diff --git a/src/Titano.Core/Import/ImportPlan.cs b/src/Titano.Core/Import/ImportPlan.cs new file mode 100644 index 00000000..806bc125 --- /dev/null +++ b/src/Titano.Core/Import/ImportPlan.cs @@ -0,0 +1,118 @@ +using System.Text.Json.Serialization; +using Titano.Core.Imaging; +using Titano.Core.Naming; + +namespace Titano.Core.Import; + +/// What to do with a photo that is already in the archive (same SHA-256). +public enum DuplicatePolicy +{ + /// Leave it on the card; the archive copy is identical. + Skip, + + /// Import it again into this session's sequence (its name gets a suffix if taken). + ImportAgain, +} + +/// Card import settings (BRIEF §12.4, §13); the defaults are the brief's. +public sealed record ImportOptions +{ + public string FolderTemplate { get; init; } = TemplateDefaults.Folder; + public string FileTemplate { get; init; } = TemplateDefaults.File; + public TimeSpan SessionGap { get; init; } = TimeSpan.FromMinutes(45); + public int MinimumSessionFrames { get; init; } = 8; + public bool IncludeSubfolders { get; init; } = true; + public bool VerifyCopies { get; init; } = true; + public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip; + + /// Card paths where the user split a session (a new session starts at that photo). + public IReadOnlySet SplitBefore { get; init; } = new HashSet(); + + /// Card paths where the user merged a session into the previous one. + public IReadOnlySet MergeAt { get; init; } = new HashSet(); + + /// Titles given to sessions, by session key; used as the sequence title and the {title} token. + public IReadOnlyDictionary Titles { get; init; } = new Dictionary(); +} + +public enum ImportAction +{ + Copy, + + /// Imported from this card before (same path, size and time). + SkipImported, + + /// Already in the archive (same size and capture time, then the same SHA-256). + SkipInArchive, +} + +/// One photo of the plan and its destination name. +public sealed record PlannedItem +{ + /// Path below the source root, forward slashes. + public required string SourcePath { get; init; } + + public long Size { get; init; } + public DateTime ModifiedUtc { get; init; } + public DateTime CapturedAt { get; init; } + public required string FileName { get; init; } + public ImportAction Action { get; init; } + + /// The archive frame this photo duplicates (, or imported again). + public Guid? ExistingFrameId { get; init; } + + /// SHA-256 computed while planning (duplicate check), if any. + public string? Sha256 { get; init; } +} + +/// A session of the plan: one camera body and file format, one sequence in the archive. +public sealed record PlannedSession +{ + /// 1-based, in order of start time across the import. + public int Number { get; init; } + + /// Stable key of the session; re-planning the same photos yields the same key, so an interrupted import + /// resumes into the same sequence. + public required string SessionKey { get; init; } + + public string? CameraMake { get; init; } + public string? CameraModel { get; init; } + public string? CameraSerial { get; init; } + public ImageFileFormat Format { get; init; } + public DateTime Start { get; init; } + public DateTime End { get; init; } + + /// Destination folder relative to the destination root, forward slashes. + public required string Folder { get; init; } + + public string? Title { get; init; } + public IReadOnlyList Items { get; init; } = []; +} + +/// +/// The exact result of an import before any byte is written (BRIEF §12.4 "Plan before touching disk"): sessions, +/// folders, file names, what is skipped and why, and the space it needs. Executing it copies exactly this. +/// +public sealed record ImportPlan +{ + public required Guid Id { get; init; } + public required string SourceRoot { get; init; } + public required string DestinationRoot { get; init; } + public DateTimeOffset Created { get; init; } + public IReadOnlyList Sessions { get; init; } = []; + public IReadOnlyList Problems { get; init; } = []; + + /// Free space at the destination when planned; null when the volume does not report it. + public long? AvailableBytes { get; init; } + + [JsonIgnore] + public IEnumerable Items => Sessions.SelectMany(s => s.Items); + [JsonIgnore] + public int FileCount => Sessions.Sum(s => s.Items.Count); + [JsonIgnore] + public int CopyCount => Items.Count(i => i.Action == ImportAction.Copy); + [JsonIgnore] + public long CopyBytes => Items.Where(i => i.Action == ImportAction.Copy).Sum(i => i.Size); + [JsonIgnore] + public bool HasEnoughSpace => AvailableBytes is not { } free || free >= CopyBytes; +} diff --git a/src/Titano.Core/Import/ImportPlanner.cs b/src/Titano.Core/Import/ImportPlanner.cs new file mode 100644 index 00000000..3def653b --- /dev/null +++ b/src/Titano.Core/Import/ImportPlanner.cs @@ -0,0 +1,181 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using Titano.Core.Archiving; +using Titano.Core.Naming; + +namespace Titano.Core.Import; + +/// An archive frame that may be the same photo. +public sealed record ArchiveMatch(Guid FrameId, string? Sha256); + +/// What the planner needs to know about the destination (implemented over the archive index). +public interface IImportLookup +{ + /// The file was imported from this card before (same path, size and time). + Task WasImportedAsync(CardFile file, CancellationToken cancellationToken); + + /// Archive frames with this size and capture time; the planner compares hashes only for these. + Task> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken); + + /// Folder of the sequence an earlier import of this session went to, so a re-import continues it. + Task FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken); + + /// The folder is taken by something else (another sequence), so the session needs another name. + bool IsFolderTaken(string folder); +} + +/// Turns card photos into an (BRIEF §12.4, §13). Nothing is written. +public static class ImportPlanner +{ + public static async Task PlanAsync(string sourceRoot, string destinationRoot, IReadOnlyList photos, + IReadOnlyList problems, ImportOptions options, IImportLookup lookup, CancellationToken cancellationToken = default) + { + var folderTemplate = PathTemplate.Parse(options.FolderTemplate); + var fileTemplate = PathTemplate.Parse(options.FileTemplate); + var drafts = new List<(string Key, List Photos)>(); + foreach (var body in photos.GroupBy(BodyKey, StringComparer.Ordinal)) + { + var ordered = body.OrderBy(p => p.CaptureTime).ThenBy(p => p.File.RelativePath, StringComparer.Ordinal).ToList(); + var ranges = SessionSplitter.Split(ordered.ConvertAll(p => p.CaptureTime), options.SessionGap, options.MinimumSessionFrames, + i => options.SplitBefore.Contains(ordered[i].File.RelativePath) ? true : options.MergeAt.Contains(ordered[i].File.RelativePath) ? false : null); + foreach (var range in ranges) + { + var members = ordered.GetRange(range.Start, range.Count); + drafts.Add((SessionKey(body.Key, members[0].CaptureTime), members)); + } + } + drafts.Sort((a, b) => a.Photos[0].CaptureTime.CompareTo(b.Photos[0].CaptureTime) is var c and not 0 ? c : string.CompareOrdinal(a.Key, b.Key)); + + var claimedFolders = new HashSet(StringComparer.OrdinalIgnoreCase); + var sessions = new List(drafts.Count); + for (var s = 0; s < drafts.Count; s++) + { + var (key, members) = drafts[s]; + var first = members[0]; + options.Titles.TryGetValue(key, out var title); + var context = Context(first, first.CaptureTime, s + 1, 1, title); + var folder = await lookup.FindSessionFolderAsync(key, cancellationToken).ConfigureAwait(false) + ?? UniqueFolder(folderTemplate.ExpandFolder(context), claimedFolders, lookup); + claimedFolders.Add(folder); + + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + var items = new List(members.Count); + for (var i = 0; i < members.Count; i++) + { + var photo = members[i]; + var name = UniqueName(fileTemplate.ExpandFile(Context(photo, first.CaptureTime, s + 1, i + 1, title)), names); + items.Add(await PlanItemAsync(sourceRoot, photo, name, options, lookup, cancellationToken).ConfigureAwait(false)); + } + sessions.Add(new PlannedSession + { + Number = s + 1, + SessionKey = key, + CameraMake = first.Metadata.Make, + CameraModel = first.Metadata.Model, + CameraSerial = first.Metadata.BodySerial, + Format = first.Metadata.Format, + Start = first.CaptureTime, + End = members[^1].CaptureTime, + Folder = folder, + Title = title, + Items = items, + }); + } + return new ImportPlan + { + Id = Guid.CreateVersion7(), + SourceRoot = Path.GetFullPath(sourceRoot), + DestinationRoot = Path.GetFullPath(destinationRoot), + Created = DateTimeOffset.UtcNow, + Sessions = sessions, + Problems = problems, + AvailableBytes = FreeSpace(destinationRoot), + }; + } + + private static async Task PlanItemAsync(string sourceRoot, CardPhoto photo, string name, ImportOptions options, IImportLookup lookup, + CancellationToken cancellationToken) + { + var item = new PlannedItem + { + SourcePath = photo.File.RelativePath, + Size = photo.File.Size, + ModifiedUtc = photo.File.ModifiedUtc, + CapturedAt = photo.CaptureTime, + FileName = name, + }; + if (await lookup.WasImportedAsync(photo.File, cancellationToken).ConfigureAwait(false)) + return item with { Action = ImportAction.SkipImported }; + var candidates = await lookup.FindCandidatesAsync(photo.File.Size, photo.CaptureTime, cancellationToken).ConfigureAwait(false); + if (candidates.Count == 0) return item; + var sha = await ContentHash.Sha256Async(Path.Combine(sourceRoot, photo.File.RelativePath), null, cancellationToken).ConfigureAwait(false); + var match = candidates.FirstOrDefault(c => sha.Equals(c.Sha256, StringComparison.OrdinalIgnoreCase)); + if (match is null) return item with { Sha256 = sha }; + return item with + { + Sha256 = sha, + ExistingFrameId = match.FrameId, + Action = options.Duplicates == DuplicatePolicy.Skip ? ImportAction.SkipInArchive : ImportAction.Copy, + }; + } + + private static NamingContext Context(CardPhoto photo, DateTime sessionStart, int session, int number, string? title) => new() + { + CapturedAt = photo.CaptureTime, + SessionStart = sessionStart, + Session = session, + Number = number, + NumberInSession = number, + CameraMake = photo.Metadata.Make, + CameraModel = photo.Metadata.Model, + Lens = photo.Metadata.LensModel, + Iso = photo.Metadata.Iso, + ExposureTime = photo.Metadata.ExposureTime, + FNumber = photo.Metadata.FNumber, + OriginalName = Path.GetFileNameWithoutExtension(photo.File.RelativePath), + Extension = Path.GetExtension(photo.File.RelativePath).TrimStart('.'), + Title = title, + }; + + /// Camera body and file format: RAW+JPEG shooting yields two sequences instead of doubled frames. + private static string BodyKey(CardPhoto p) => $"{p.Metadata.Make}|{p.Metadata.Model}|{p.Metadata.BodySerial}|{p.Metadata.Format}"; + + private static string SessionKey(string bodyKey, DateTime start) => + "card-" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(bodyKey)))[..8] + "-" + + start.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture); + + private static string UniqueFolder(string folder, HashSet claimed, IImportLookup lookup) + { + if (folder.Length == 0) folder = "Import"; + var candidate = folder; + for (var n = 2; claimed.Contains(candidate) || lookup.IsFolderTaken(candidate); n++) + candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})"); + return candidate; + } + + /// Adds _2, _3… before the extension when the template gives two photos the same name. + internal static string UniqueName(string name, HashSet taken) + { + var candidate = name; + var stem = Path.GetFileNameWithoutExtension(name); + var extension = Path.GetExtension(name); + for (var n = 2; !taken.Add(candidate); n++) candidate = string.Create(CultureInfo.InvariantCulture, $"{stem}_{n}{extension}"); + return candidate; + } + + private static long? FreeSpace(string path) + { + try + { + var full = Path.GetFullPath(path); + while (!Directory.Exists(full) && Path.GetDirectoryName(full) is { } parent) full = parent; + // Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on. + return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace; + } + catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException) + { + return null; + } + } +} diff --git a/src/Titano.Core/Import/SessionSplitter.cs b/src/Titano.Core/Import/SessionSplitter.cs new file mode 100644 index 00000000..c241fc24 --- /dev/null +++ b/src/Titano.Core/Import/SessionSplitter.cs @@ -0,0 +1,61 @@ +namespace Titano.Core.Import; + +/// A run of consecutive frames forming one session: [Start, Start + Count). +public readonly record struct SessionRange(int Start, int Count) +{ + public int End => Start + Count; +} + +/// +/// Splits a time-ordered run of captures into shooting sessions (BRIEF §13): a pause longer than the gap starts a +/// new session; a session with fewer than the minimum frames (test shots) merges into the previous one, or into the +/// next when it comes first. Breaks the user forced are never merged away; breaks the user removed never split. +/// +public static class SessionSplitter +{ + /// Capture times in ascending order. + /// For a frame index: true = the user starts a session there, false = the user joined it + /// to the previous frame, null = automatic. + public static IReadOnlyList Split(IReadOnlyList times, TimeSpan gap, int minimumFrames, Func? userBreak = null) + { + if (times.Count == 0) return []; + var groups = new List<(int Start, int Count, bool Forced)>(); + var start = 0; + var forced = false; + for (var i = 1; i < times.Count; i++) + { + var decision = userBreak?.Invoke(i); + if (decision == false || (decision is null && times[i] - times[i - 1] <= gap)) continue; + groups.Add((start, i - start, forced)); + start = i; + forced = decision == true; + } + groups.Add((start, times.Count - start, forced)); + + for (var i = 0; i < groups.Count && groups.Count > 1;) + { + var g = groups[i]; + if (g.Count >= minimumFrames) + { + i++; + continue; + } + if (i > 0 && !g.Forced) + { + var previous = groups[i - 1]; + groups[i - 1] = (previous.Start, previous.Count + g.Count, previous.Forced); + groups.RemoveAt(i); + } + else if (i == 0 && !groups[1].Forced) + { + groups[1] = (g.Start, g.Count + groups[1].Count, g.Forced); + groups.RemoveAt(0); + } + else + { + i++; + } + } + return groups.ConvertAll(g => new SessionRange(g.Start, g.Count)); + } +} diff --git a/src/Titano.Core/Import/VerifiedCopy.cs b/src/Titano.Core/Import/VerifiedCopy.cs new file mode 100644 index 00000000..14fceb9c --- /dev/null +++ b/src/Titano.Core/Import/VerifiedCopy.cs @@ -0,0 +1,65 @@ +using System.Security.Cryptography; +using Titano.Core.Archiving; +using Titano.Core.Localization; + +namespace Titano.Core.Import; + +/// The copy read back from the destination does not match what was read from the source. +public sealed class CopyVerificationException(string path) + : IOException(Loc.L("Archive.Error.CopyMismatch", Path.GetFileName(path))) +{ + public string FilePath { get; } = path; +} + +/// +/// Copies a file while hashing it, flushes the copy to disk, then re-reads it and compares hashes (BRIEF §12.4). +/// A failed or interrupted copy never leaves a partial file behind. +/// +public static class VerifiedCopy +{ + /// Copies to the new file ; returns the SHA-256. + /// Re-read the copy and compare (the default import setting). + /// Receives the bytes of each chunk read from the source. + /// The re-read copy differs. + public static async Task CopyAsync(string source, string destination, bool verify, Action? bytesCopied = null, + CancellationToken cancellationToken = default) + { + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(destination))!); + var created = false; + try + { + string sha; + DateTime modified; + await using (var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, ContentHash.BufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan)) + await using (var output = new FileStream(destination, FileMode.CreateNew, FileAccess.Write, FileShare.None, ContentHash.BufferSize, + FileOptions.Asynchronous)) + { + created = true; + modified = File.GetLastWriteTimeUtc(source); + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var buffer = new byte[ContentHash.BufferSize]; + int read; + while ((read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + hash.AppendData(buffer, 0, read); + await output.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + bytesCopied?.Invoke(read); + } + await output.FlushAsync(cancellationToken).ConfigureAwait(false); + output.Flush(flushToDisk: true); + sha = Convert.ToHexStringLower(hash.GetHashAndReset()); + } + File.SetLastWriteTimeUtc(destination, modified); + if (verify && await ContentHash.Sha256Async(destination, null, cancellationToken).ConfigureAwait(false) != sha) + throw new CopyVerificationException(destination); + return sha; + } + catch + { + // Only our own partial or unverified copy; an existing file at the destination is never touched. + if (created && File.Exists(destination)) File.Delete(destination); + throw; + } + } +} diff --git a/src/Titano.Data/Archiving/ArchiveImages.cs b/src/Titano.Data/Archiving/ArchiveImages.cs new file mode 100644 index 00000000..e8ae63be --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveImages.cs @@ -0,0 +1,174 @@ +using System.Diagnostics; +using Titano.Core.Archiving; +using Titano.Core.IO; +using Titano.Core.Metadata; +using Titano.Core.Model; + +namespace Titano.Data.Archiving; + +/// Frames processed by one thumbnail or proxy pass over a sequence. +public sealed record ImagePassResult(int Built, int Reused, int Undecodable); + +/// +/// Thumbnails (thumbs.pack) and proxies (proxies/0001.jpg) of sequences (BRIEF §8.5). Both are +/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after +/// frames were renumbered the stale ones are rebuilt and the rest reused. +/// +public sealed class ArchiveImages +{ + private const int CommitFrames = 200; + private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10); + + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + private readonly FrameImageRenderer _renderer; + + internal ArchiveImages(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + _renderer = new FrameImageRenderer(writer.Options.Decoder); + } + + /// Resume point inside the sequence (frames before it were done by an earlier run). + /// Called with the next frame index whenever the pack has been committed. + public async Task BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame, + Func checkpoint, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(sequence.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + int built = 0, reused = 0, undecodable = 0; + // Thumbnail writers exclude each other only; ingest keeps committing meanwhile. + using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait, + _writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false)) + using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath)) + { + var cadence = new Cadence(); + for (var i = startFrame; i < manifest.Frames.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var frame = manifest.Frames[i]; + var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size); + if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint)) + { + reused++; + continue; + } + if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image) + { + pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg); + built++; + } + else + { + undecodable++; + } + if (cadence.Due()) + { + pack.Commit(); + await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); + } + } + pack.Commit(); + } + await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false); + return new ImagePassResult(built, reused, undecodable); + } + + /// + public async Task BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame, + Func checkpoint, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy; + if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0); + var folder = paths.Sequence(sequence.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + Directory.CreateDirectory(folder.Proxies); + int built = 0, reused = 0, undecodable = 0; + var cadence = new Cadence(); + for (var i = startFrame; i < manifest.Frames.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var frame = manifest.Frames[i]; + var path = folder.Proxy(frame.I); + if (!force && IsCurrent(path, frame)) + { + reused++; + continue; + } + if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image) + { + AtomicFile.WriteAllBytes(path, image.Jpeg); + built++; + } + else + { + undecodable++; + } + if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false); + } + await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false); + return new ImagePassResult(built, reused, undecodable); + } + + /// A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale). + private static bool IsCurrent(string proxy, SidecarFrame frame) + { + if (!File.Exists(proxy)) return false; + try + { + return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// A frame no decoder can read is skipped and counted; it never stops the pass. + private static EncodedImage? Render(Func render) + { + try + { + return render(); + } + catch (Exception ex) when (ex is IOException or InvalidDataException or NotSupportedException) + { + Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex); + return null; + } + } + + /// Commit/checkpoint rhythm: every rendered frames or . + private sealed class Cadence + { + private readonly Stopwatch _clock = Stopwatch.StartNew(); + private int _frames; + + public bool Due() + { + if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false; + _frames = 0; + _clock.Restart(); + return true; + } + } + + private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken) + { + await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => + { + var status = complete ? m.Status | flag : m.Status & ~flag; + return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow }; + }, cancellationToken).ConfigureAwait(false); + await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken) + .ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Titano.Data/Archiving/ArchiveIndexer.cs b/src/Titano.Data/Archiving/ArchiveIndexer.cs new file mode 100644 index 00000000..84cc6266 --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveIndexer.cs @@ -0,0 +1,268 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Model; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +public enum ScanMode +{ + /// Only folders whose sidecar or photo folder changed size or time since the last scan. + Incremental, + + /// Every folder: sidecars re-read and folders reconciled with their photos (manual rescan). + Full, + + /// The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5). + Rebuild, +} + +public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed); + +/// Folders that could not be indexed, with the reason; they are retried by the next scan. +public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed); + +/// +/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts +/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one +/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived. +/// +public sealed class ArchiveIndexer +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + + internal ArchiveIndexer(TitanoDatabase db, SequenceWriter writer) + { + _db = db; + _writer = writer; + } + + /// Last folder completed by an interrupted run (job checkpoint), or null. + /// Called after each folder; persist as the resume point. + /// The archive is offline or read-only; the index is left as it is. + public async Task ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func progress, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken); + var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)); + var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null; + using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null; + + int indexed = 0, adopted = 0; + var failed = new List<(string, string)>(); + for (var i = 0; i < folders.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var folder = folders[i]; + if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue; + states.TryGetValue(folder.RelativeFolder, out var state); + var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos; + if (changed) + { + try + { + var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos); + var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false); + if (snapshotWriter is not null) + { + var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath; + snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value, + bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false)); + } + indexed++; + if (folder.Sidecar is null) adopted++; + } + catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) + { + Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex); + failed.Add((folder.RelativeFolder, ex.Message)); + } + } + await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false); + } + + var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0; + if (failed.Count == 0) snapshotWriter?.Commit(); + return new ScanSummary(folders.Count, indexed, adopted, removed, failed); + } + + /// Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps. + internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken) + { + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false); + foreach (var key in manifest.SessionKeys) + await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + + /// + /// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next + /// incremental scan does not re-read what is already indexed. + /// + internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) => + _db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId, + FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken); + + /// Indexes unindexed photos and flags missing files of one sequence (the orphans fix action). + public async Task ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var folder = paths.Sequence(sequence.Folder); + var manifest = await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceAdopter.Reconcile(folder, m, cancellationToken), + cancellationToken).ConfigureAwait(false); + await SyncAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false); + return manifest; + } + + /// Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot). + private async Task IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot, + Locations locations, CancellationToken cancellationToken) + { + var sequence = paths.Sequence(folder.RelativeFolder); + SequenceManifest manifest; + byte[]? bytes = null; + if (folder.Sidecar is not { } stamp) + { + manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false); + } + else + { + bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await File.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false); + try + { + manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath); + } + catch (InvalidDataException ex) + { + Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex); + manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false); + bytes = null; + } + } + if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder)) + { + manifest = Reidentify(manifest); + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + bytes = null; + } + if (reconcile || File.Exists(sequence.IngestJournalPath)) + { + var before = manifest; + manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken), + cancellationToken).ConfigureAwait(false); + if (!ReferenceEquals(before, manifest)) bytes = null; + } + await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + locations.Set(manifest.Id, folder.RelativeFolder); + return bytes; + } + + /// A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept. + private async Task AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken) + { + var known = locations.AtFolder(sequence.RelativeFolder); + var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken), + sequence, cancellationToken).ConfigureAwait(false); + Directory.CreateDirectory(sequence.MetaDirectory); + // Another process may adopt the same folder at the same moment: the first sidecar wins. + return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath); + } + + /// Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos. + private async Task ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken) + { + File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}"); + return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false); + } + + private async Task WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken) + { + var known = new Dictionary(StringComparer.Ordinal); + var prefix = sequence.ArchiveRelative(string.Empty); + for (var after = -1; ;) + { + var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false); + foreach (var f in page) + { + if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id; + } + if (page.Count < 1000) break; + after = page[^1].Index; + } + if (known.Count == 0) return manifest; + return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() }; + } + + /// A copied sequence folder carries the ids of its source; the copy gets ids of its own. + private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with + { + Id = Guid.CreateVersion7(), + Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(), + SessionKeys = [], + UpdatedAt = DateTimeOffset.UtcNow, + }; + + /// Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched). + private async Task RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList folders, Locations locations, + CancellationToken cancellationToken) + { + ArchiveVolume.EnsureWritable(paths); + var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal); + var removed = 0; + foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + { + if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue; + if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++; + await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false); + } + return removed; + } + + /// Which folder each indexed sequence id lives in, updated as the scan goes. + private sealed class Locations + { + private readonly Dictionary _byId = []; + private readonly Dictionary _byFolder = new(StringComparer.Ordinal); + + public Locations(IEnumerable rows) + { + foreach (var row in rows) Set(row.Id, row.Folder); + } + + public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null; + + /// The id is indexed at another folder whose sidecar still carries it: this folder is a copy. + public bool IsCopy(ArchivePaths paths, Guid id, string folder) + { + if (!_byId.TryGetValue(id, out var other) || other == folder) return false; + var otherSidecar = paths.Sequence(other).ManifestPath; + try + { + return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id; + } + catch (InvalidDataException) + { + return false; + } + } + + public void Set(Guid id, string folder) + { + if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous); + _byId[id] = folder; + _byFolder[folder] = id; + } + } +} diff --git a/src/Titano.Data/Archiving/ArchiveIngest.cs b/src/Titano.Data/Archiving/ArchiveIngest.cs new file mode 100644 index 00000000..29a31c2a --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveIngest.cs @@ -0,0 +1,344 @@ +using System.Collections.Concurrent; +using System.Globalization; +using Titano.Core.Archiving; +using Titano.Core.Localization; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Core.Naming; + +namespace Titano.Data.Archiving; + +/// A verified staged file and how to name it. +/// Name chosen by an import plan; null = the archive's file template. +internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata, + DateTime? ClientCapturedAt, string? PlannedName); + +/// A sequence found or created by key (upload session, card session, or a new title). +/// Folder of an import plan; null = the archive's folder template. +internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy); + +/// +/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder +/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits +/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps +/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose +/// SHA-256 the sequence already holds is reported as a duplicate and left in staging. +/// +public sealed class ArchiveIngest : IArchiveIngest +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + private readonly ConcurrentDictionary _queues = new(); + private readonly SemaphoreSlim _creation = new(1, 1); + + internal ArchiveIngest(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + public async Task CommitAsync(IngestRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1) + throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request)); + var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request)); + var staged = new FileInfo(request.StagedPath); + if (!staged.Exists || staged.Length != request.Size) + throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath); + var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256, + MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null); + + Guid sequenceId; + if (request.SequenceId is { } id) + { + var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false); + if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request)); + sequenceId = id; + } + else + { + // A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one. + var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}"; + sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()), + frame, cancellationToken).ConfigureAwait(false); + } + return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0]; + } + + /// Commits files of one import session into the sequence of its key (created on first use). + internal async Task> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList frames, + CancellationToken cancellationToken) + { + if (frames.Count == 0) return []; + var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false); + return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false); + } + + private async Task FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken) + { + if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found; + await _creation.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again; + return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false); + } + finally + { + _creation.Release(); + } + } + + private async Task CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var now = DateTimeOffset.UtcNow; + var m = first.Metadata; + var manifest = new SequenceManifest + { + Id = Guid.CreateVersion7(), + Title = target.Title, + Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial }, + Lens = m.LensModel, + SessionKeys = [target.Key], + CreatedBy = target.CreatedBy, + Created = now, + UpdatedAt = now, + }; + var captured = CaptureTime(first); + var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title)); + if (wanted.Length == 0) wanted = "Sequence"; + var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)) + .Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase); + var sequence = ClaimFolder(paths, wanted, taken, manifest); + + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false); + var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false); + if (winner != manifest.Id) + { + // Another process bound the key first: its sequence wins, ours was never used. + await tx.RollbackAsync(cancellationToken).ConfigureAwait(false); + ReleaseFolder(sequence); + return winner; + } + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + return manifest.Id; + } + + /// The first free variant of , claimed by creating its sidecar (first writer wins). + private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet taken, SequenceManifest manifest) + { + for (var n = 1; ; n++) + { + var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})"); + if (taken.Contains(folder)) continue; + var sequence = paths.Sequence(folder); + if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue; + Directory.CreateDirectory(sequence.MetaDirectory); + if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence; + } + } + + /// Removes the empty sequence this process created and lost to a concurrent creator. + private static void ReleaseFolder(SequencePaths sequence) + { + File.Delete(sequence.ManifestPath); + if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory); + if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder); + } + + /// Group commit: whoever gets the sequence's gate commits everything queued for it. + private async Task EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList frames, CancellationToken cancellationToken) + { + var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue()); + var pending = frames.Select(f => new Pending(f)).ToList(); + foreach (var p in pending) queue.Items.Enqueue(p); + await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (pending.Any(p => !p.Completion.Task.IsCompleted)) + { + var batch = new List(); + while (queue.Items.TryDequeue(out var p)) batch.Add(p); + try + { + await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + foreach (var p in batch) p.Completion.TrySetException(ex); + } + } + } + finally + { + queue.Gate.Release(); + } + return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false); + } + + private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List batch, CancellationToken cancellationToken) + { + var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed."); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var sequence = paths.Sequence(row.Folder); + await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false)) + { + var manifest = SequenceWriter.ReadLocked(sequence, sequenceId); + var before = manifest.Frames.Count; + var known = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var f in manifest.Frames) + { + if (f.Sha256 is not null) known.TryAdd(f.Sha256, f); + } + + var accepted = new List<(Pending Pending, SidecarFrame Frame)>(); + var duplicates = new List<(Pending Pending, Guid FrameId)>(); + foreach (var p in batch) + { + if (known.TryGetValue(p.Frame.Sha256, out var existing)) + { + duplicates.Add((p, existing.Id)); + continue; + } + var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size, + p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt); + known[p.Frame.Sha256] = frame; + accepted.Add((p, frame)); + } + + var named = Name(archive, sequence, manifest, accepted); + IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList())); + var moved = new List(); + foreach (var (p, frame) in accepted) + { + var final = named[frame.Id]; + try + { + Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!); + File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false); + moved.Add(final); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + p.Completion.TrySetException(ex); + } + } + var updated = SequenceFrames.Insert(manifest, moved); + if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated); + IngestJournal.Delete(sequence); + + var byId = updated.Frames.ToDictionary(f => f.Id); + if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false); + foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id))) + p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false)); + foreach (var (p, frameId) in duplicates) + { + if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true)); + else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName))); + } + } + } + + /// Final file names, in capture order: the plan's name or the template with {n} = index + 1, made unique. + private static Dictionary Name(Archive archive, SequencePaths sequence, SequenceManifest manifest, + List<(Pending Pending, SidecarFrame Frame)> accepted) + { + var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame)); + var incoming = accepted.ToDictionary(a => a.Frame.Id); + // Photos adopted in place stay in place; everything else goes to originals/. + var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/')) + ? ArchiveLayout.OriginalsFolder + "/" + : string.Empty; + var taken = new HashSet(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase); + var onDisk = Path.Combine(sequence.Folder, directory); + if (Directory.Exists(onDisk)) + { + foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file)); + } + var template = PathTemplate.Parse(archive.FileTemplate); + var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null; + var named = new Dictionary(); + foreach (var frame in merged.Frames) + { + if (!incoming.TryGetValue(frame.Id, out var entry)) continue; + var staged = entry.Pending.Frame; + var captured = frame.CapturedAt ?? DateTime.Now; + var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title)); + var file = directory + name; + var stem = Path.GetFileNameWithoutExtension(name); + for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}"); + named[frame.Id] = entry.Frame with { File = file }; + } + return named; + } + + /// Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence. + private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List appended, + CancellationToken cancellationToken) + { + var inStep = appended.All(f => f.I >= before) + && await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before; + if (!inStep) + { + await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false); + return; + } + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false); + } + + private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) => + new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate); + + private static DateTime CaptureTime(StagedFrame frame) => + frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client + ? client + : frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now; + + private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new() + { + CapturedAt = captured, + SessionStart = sessionStart, + Number = number, + NumberInSession = number, + CameraMake = frame.Metadata.Make, + CameraModel = frame.Metadata.Model, + Lens = frame.Metadata.LensModel, + Iso = frame.Metadata.Iso, + ExposureTime = frame.Metadata.ExposureTime, + FNumber = frame.Metadata.FNumber, + OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName), + Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'), + Title = title, + }; + + private sealed class CommitQueue + { + public SemaphoreSlim Gate { get; } = new(1, 1); + public ConcurrentQueue Items { get; } = new(); + } + + private sealed class Pending(StagedFrame frame) + { + public StagedFrame Frame { get; } = frame; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } +} diff --git a/src/Titano.Data/Archiving/ArchiveMaintenance.cs b/src/Titano.Data/Archiving/ArchiveMaintenance.cs new file mode 100644 index 00000000..7e598f2d --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveMaintenance.cs @@ -0,0 +1,222 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Model; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +/// Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original. +public sealed record DuplicateGroup(string Sha256, IReadOnlyList Frames); + +/// Index and disk out of step (BRIEF §12.5 Orphans). +/// Per indexed sequence: photos the sidecar does not list and frames whose file is gone. +/// Index rows whose folder holds no sidecar any more. +/// Folders with photos or a sidecar that the index does not know yet (a scan adopts them). +public sealed record OrphanReport(IReadOnlyList Sequences, IReadOnlyList MissingSequences, IReadOnlyList UnindexedFolders) +{ + public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count); + public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count); +} + +/// What a maintenance run removed or changed (or would, in a dry run). +public sealed record MaintenanceReport( + bool DryRun, + IReadOnlyList StaleLeases, + IReadOnlyList TemporaryFiles, + IReadOnlyList CompactedPacks, + int DuplicateGroups, + int DuplicateFlagChanges, + int UnindexedFiles, + int MissingFiles); + +/// Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched. +public sealed class ArchiveMaintenance +{ + /// Staged copies older than this belong to no running import. + private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24); + + /// Temporary files of atomic writes older than this were left by a crash. + private static readonly TimeSpan TempAge = TimeSpan.FromHours(1); + + /// Packs with more reclaimable bytes than this share are compacted. + private const double CompactionRatio = 0.3; + + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + + internal ArchiveMaintenance(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + public async Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + (await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false)) + .GroupBy(f => f.Sha256, StringComparer.Ordinal) + .Select(g => new DuplicateGroup(g.Key, g.ToList())) + .ToList(); + + public async Task FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline); + var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var sequences = new List(); + var missing = new List(); + foreach (var location in locations) + { + cancellationToken.ThrowIfCancellationRequested(); + var sequence = paths.Sequence(location.Folder); + if (!File.Exists(sequence.ManifestPath)) + { + missing.Add(location); + continue; + } + var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath)); + if (!orphans.IsEmpty) sequences.Add(orphans); + } + var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal); + var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList(); + return new OrphanReport(sequences, missing, unindexed); + } + + /// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept. + public async Task RemoveMissingSequencesAsync(Archive archive, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline); + var removed = 0; + foreach (var id in sequenceIds) + { + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue; + if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue; + if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++; + await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false); + } + return removed; + } + + /// Number of maintenance steps (the job checkpoints after each). + internal const int StepCount = 4; + + /// Runs the maintenance steps from : stale leases, leftover temporary files, + /// thumbnail pack compaction, duplicate flags. A dry run only reports. + public async Task RunAsync(Archive archive, bool dryRun, int firstStep, Func stepDone, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + if (!dryRun) ArchiveVolume.EnsureWritable(paths); + var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + IReadOnlyList leases = [], temps = [], packs = []; + int groups = 0, flagChanges = 0; + for (var step = firstStep; step < StepCount; step++) + { + cancellationToken.ThrowIfCancellationRequested(); + switch (step) + { + case 0: + leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun); + break; + case 1: + temps = PurgeTemporaryFiles(paths, locations, dryRun); + break; + case 2: + packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false); + break; + case 3: + (groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false); + break; + } + await stepDone(step + 1).ConfigureAwait(false); + } + var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false); + return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles); + } + + /// Staging copies and atomic-write temporaries left behind by crashes (never originals). + private static List PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList locations, bool dryRun) + { + var now = DateTime.UtcNow; + var found = new List(); + void Collect(string directory, string pattern, TimeSpan age, bool recursive) + { + if (!Directory.Exists(directory)) return; + foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)) + { + if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file); + } + } + Collect(paths.StagingDirectory, "*", StagingAge, recursive: true); + Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false); + foreach (var location in locations) + { + var sequence = paths.Sequence(location.Folder); + Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false); + Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false); + } + if (!dryRun) + { + foreach (var file in found) + { + try + { + File.Delete(file); + } + catch (IOException ex) + { + Log.Warning("archive", $"Could not remove {file}", ex); + } + } + } + return found.ConvertAll(paths.ToRelative); + } + + private async Task> CompactPacksAsync(ArchivePaths paths, IReadOnlyList locations, bool dryRun, CancellationToken cancellationToken) + { + var compacted = new List(); + foreach (var location in locations) + { + var sequence = paths.Sequence(location.Folder); + long garbage, length; + using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath)) + { + garbage = reader.GarbageBytes; + length = reader.Length; + } + if (length == 0 || garbage < length * CompactionRatio) continue; + compacted.Add(location.Folder); + if (dryRun) continue; + await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false)) + ThumbPackWriter.Compact(sequence.ThumbsPackPath); + } + return compacted; + } + + /// Sets on every copy after the first of each hash and clears it elsewhere. + private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken) + { + var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId); + var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false)) + .ToDictionary(f => f.FrameId, f => f.SequenceId); + var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList(); + var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList(); + if (dryRun) return (groups.Count, set.Count + clear.Count); + + foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value)) + { + if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue; + var changes = bySequence.ToDictionary(x => x.Key, x => (Func)(f => + f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate })); + await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); + var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList(); + var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList(); + if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false); + if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false); + } + return (groups.Count, set.Count + clear.Count); + } +} diff --git a/src/Titano.Data/Archiving/ArchiveService.cs b/src/Titano.Data/Archiving/ArchiveService.cs new file mode 100644 index 00000000..9463c422 --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveService.cs @@ -0,0 +1,232 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Import; +using Titano.Core.Model; +using Titano.Data.Archiving.Jobs; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving; + +/// +/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create +/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and +/// the job handlers for the long operations (enqueue them with ). +/// +public sealed class ArchiveService +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + + public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null) + { + _db = db; + _writer = new SequenceWriter(options ?? new ArchiveServiceOptions()); + Indexer = new ArchiveIndexer(db, _writer); + Ingest = new ArchiveIngest(db, _writer, Indexer); + Verifier = new ArchiveVerifier(db, _writer, Indexer); + Maintenance = new ArchiveMaintenance(db, _writer, Indexer); + Renamer = new MassRenamer(db, _writer, Indexer); + Images = new ArchiveImages(db, _writer, Indexer); + Imports = new ImportService(db, Ingest); + } + + public ArchiveIndexer Indexer { get; } + + /// Commits staged files into sequences (server uploads, client card imports). + public ArchiveIngest Ingest { get; } + + public ArchiveVerifier Verifier { get; } + public ArchiveMaintenance Maintenance { get; } + public MassRenamer Renamer { get; } + public ArchiveImages Images { get; } + public ImportService Imports { get; } + + /// Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the . + public IReadOnlyList CreateJobHandlers() => + [ + new ScanJobHandler(_db, Indexer, JobKind.Scan), + new ScanJobHandler(_db, Indexer, JobKind.Rebuild), + new VerifyJobHandler(_db, Verifier), + new ImportJobHandler(Imports), + new ImageJobHandler(_db, Images, JobKind.Thumbs), + new ImageJobHandler(_db, Images, JobKind.Proxies), + new MaintenanceJobHandler(_db, Maintenance), + ]; + + /// Makes a new, empty archive and indexes it. + /// The folder already is an archive (open it instead). + public async Task CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(root); + if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive."); + Directory.CreateDirectory(paths.MetaDirectory); + var manifest = new ArchiveManifest + { + Id = Guid.CreateVersion7(), + Name = name, + Created = DateTimeOffset.UtcNow, + LayoutTemplates = templates ?? new LayoutTemplates(), + ProxyPolicy = proxies ?? new ProxyPolicy(), + }; + ManifestStore.WriteArchive(paths.ManifestPath, manifest); + return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false); + } + + /// + /// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash + /// interrupted. An offline archive is registered as it was last seen. + /// + public async Task OpenAsync(string root, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(root); + var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false); + if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive; + try + { + await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException) + { + // The journal stays; the next open tries again. The archive itself is usable meanwhile. + Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex); + } + return archive; + } + + /// + /// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue + /// with a full pass to write sidecars and index it. + /// + public Task AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(root); + return File.Exists(paths.ManifestPath) + ? OpenAsync(root, cancellationToken) + : CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken); + } + + public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath)); + + /// Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1). + public async Task SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList tags, int rating, string? notes, + CancellationToken cancellationToken = default) + { + var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with + { + Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(), + Tags = tags, + Rating = rating, + Notes = notes, + Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }, cancellationToken).ConfigureAwait(false); + var folder = paths.Sequence(sequence.Folder); + var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence); + await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false); + await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false); + return updated; + } + + public Task> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + Maintenance.FindDuplicatesAsync(archiveId, cancellationToken); + + public async Task FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) => + await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false); + + /// Orphans fix: index the unlisted photos of a sequence and flag its missing files. + public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default) + { + var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false); + await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false); + } + + /// Orphans fix: drop index rows whose sidecar is gone. + public async Task RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList sequenceIds, CancellationToken cancellationToken = default) => + await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken) + .ConfigureAwait(false); + + /// Dry run of a mass rename. + public async Task PlanRenameAsync(Guid archiveId, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate, + CancellationToken cancellationToken = default) => + await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate, + cancellationToken).ConfigureAwait(false); + + public async Task ApplyRenameAsync(RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => + await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken) + .ConfigureAwait(false); + + /// Plans a card or folder import into an archive (nothing is copied). + public async Task PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress? progress = null, + CancellationToken cancellationToken = default) => + await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress, + cancellationToken).ConfigureAwait(false); + + /// Dry run of "delete imported" for a card; pass the plan to after the user confirms. + public async Task PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress? progress = null, + CancellationToken cancellationToken = default) => + await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress, + cancellationToken).ConfigureAwait(false); + + private async Task RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken) + { + var archive = new Archive + { + Id = manifest.Id, + Name = manifest.Name, + RootPath = paths.Root, + FolderTemplate = manifest.LayoutTemplates.Folder, + FileTemplate = manifest.LayoutTemplates.File, + SchemaVersion = manifest.SchemaVersion, + Created = manifest.Created, + ProxyLongEdge = manifest.ProxyPolicy.LongEdge, + }; + await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false); + return archive; + } + + private async Task LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) => + await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId)); + + private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken) + { + var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) + ?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId)); + return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence); + } +} + +/// Builds the archive jobs; enqueue with . +public static class ArchiveJobs +{ + public static ArchiveJobRequest Scan(Archive archive, bool full = false) => + new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User); + + public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User); + + public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList? sequenceIds = null) => + new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User); + + public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) => + new(JobKind.Import, [archive.Id], archive.Name, + Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User); + + public static ArchiveJobRequest Thumbnails(IReadOnlyList sequenceIds, string? label = null, bool force = false) => + new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background); + + public static ArchiveJobRequest Proxies(IReadOnlyList sequenceIds, string? label = null, bool force = false) => + new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background); + + public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) => + new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters), + JobPriority.Normal); + + public static Task EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) => + engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken); + + private static string Json(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo) => JsonSerializer.Serialize(value, typeInfo); +} diff --git a/src/Titano.Data/Archiving/ArchiveVerifier.cs b/src/Titano.Data/Archiving/ArchiveVerifier.cs new file mode 100644 index 00000000..db6bcd8c --- /dev/null +++ b/src/Titano.Data/Archiving/ArchiveVerifier.cs @@ -0,0 +1,127 @@ +using System.Diagnostics; +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +/// A frame whose original is missing, changed or damaged. +public sealed record VerifyProblem(Guid SequenceId, Guid FrameId, string RelativePath, FrameHealth Health); + +/// Where a verify stands: the next sequence (ordinal in the run) and the next frame index in it. +public readonly record struct VerifyPosition(int Sequence, int Frame); + +/// Outcome of one verify run (frames before a resume point are counted by the run that checked them). +public sealed record VerifyReport(int Sequences, long Checked, long Ok, long Hashed, IReadOnlyList Problems); + +/// +/// Re-hashes originals against their sidecar SHA-256 (BRIEF §12.5 Verify): missing, changed and corrupt files get +/// / in the sidecar and the index, adopted files get +/// their first hash, and a clean sequence is stamped verified. Progress is flushed in batches so a resumed run +/// continues where the last flush ended. +/// +public sealed class ArchiveVerifier +{ + private const int FlushFrames = 512; + private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30); + + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + + internal ArchiveVerifier(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + /// Sequences to verify; null = the whole archive (ordered by folder). + /// Called after each flush with the resume position, bytes done and bytes total. + /// Called after each frame with bytes done and bytes total (throughput and ETA). + public async Task VerifyAsync(Archive archive, IReadOnlyList? sequenceIds, VerifyPosition resume, + Func checkpoint, Func? frameDone = null, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false); + var bytesTotal = targets.Sum(t => t.TotalBytes); + var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes); + long checkedFrames = 0, ok = 0, hashed = 0; + var problems = new List(); + for (var s = resume.Sequence; s < targets.Count; s++) + { + var target = targets[s]; + var sequence = paths.Sequence(target.Folder); + var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); + var pending = new List(); + var clock = Stopwatch.StartNew(); + var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0; + bytesDone += manifest.Frames.Take(first).Sum(f => f.Size); + for (var i = first; i < manifest.Frames.Count; i++) + { + cancellationToken.ThrowIfCancellationRequested(); + var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false); + checkedFrames++; + if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false); + if (check.Health == FrameHealth.Ok) ok++; + else if (check.Health == FrameHealth.Hashed) hashed++; + else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health)); + if (check.ChangesRecord) pending.Add(check); + if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval)) + { + await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); + await checkpoint(new VerifyPosition(s, i + 1), bytesDone, bytesTotal).ConfigureAwait(false); + clock.Restart(); + } + } + await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false); + await StampAsync(archive, paths, target, cancellationToken).ConfigureAwait(false); + await checkpoint(new VerifyPosition(s + 1, 0), bytesDone, bytesTotal).ConfigureAwait(false); + } + return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems); + } + + private async Task> TargetsAsync(Archive archive, IReadOnlyList? sequenceIds, CancellationToken cancellationToken) + { + var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList(); + var targets = new List(ids.Count); + foreach (var id in ids) + { + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s); + } + return targets; + } + + /// Writes the flags and new hashes of checked frames to the sidecar and the index. + private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List pending, CancellationToken cancellationToken) + { + if (pending.Count == 0) return; + var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func)(f => + f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 })); + await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false); + await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)), + cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false); + pending.Clear(); + } + + /// Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt. + private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken) + { + var sequence = paths.Sequence(target.Folder); + var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m => + { + var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0); + return m with + { + Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified, + VerifiedAt = DateTimeOffset.UtcNow, + UpdatedAt = DateTimeOffset.UtcNow, + }; + }, cancellationToken).ConfigureAwait(false); + var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false); + await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/Titano.Data/Archiving/ImportService.cs b/src/Titano.Data/Archiving/ImportService.cs new file mode 100644 index 00000000..e7db6a6e --- /dev/null +++ b/src/Titano.Data/Archiving/ImportService.cs @@ -0,0 +1,219 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Core.Naming; +using Titano.Data.Archiving.Jobs; +using Titano.Data.Repositories; + +namespace Titano.Data.Archiving; + +/// A card volume as the client sees it (BRIEF §13): identity {serial}-{label}-{capacity}. +public sealed record CardVolume(string Identity, uint SerialNumber, string? Label, long Capacity); + +public enum AfterImport +{ + Keep, + + /// Delete from the card the photos whose card file and archive copy both re-hash to the imported hash. + DeleteImported, + + /// Offer the operating system's format dialog (only after a fully verified import). + FormatCard, +} + +/// Import defaults remembered per card (, BRIEF §12.4). +public sealed record CardProfile +{ + public string FolderTemplate { get; init; } = TemplateDefaults.Folder; + public string FileTemplate { get; init; } = TemplateDefaults.File; + public int SessionGapMinutes { get; init; } = 45; + public int MinimumSessionFrames { get; init; } = 8; + public bool IncludeSubfolders { get; init; } = true; + public bool VerifyCopies { get; init; } = true; + public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip; + public AfterImport AfterImport { get; init; } = AfterImport.Keep; + public bool Eject { get; init; } = true; + public bool DeepAnalysis { get; init; } = true; + + public ImportOptions ToOptions() => new() + { + FolderTemplate = FolderTemplate, + FileTemplate = FileTemplate, + SessionGap = TimeSpan.FromMinutes(SessionGapMinutes), + MinimumSessionFrames = MinimumSessionFrames, + IncludeSubfolders = IncludeSubfolders, + VerifyCopies = VerifyCopies, + Duplicates = Duplicates, + }; +} + +/// +/// Card import over the archive index (BRIEF §12.4, §13): card profiles, planning with the archive's history and +/// duplicates, execution into the archive through , and the verified card erase. +/// +public sealed class ImportService +{ + private readonly TitanoDatabase _db; + private readonly ArchiveIngest _ingest; + + internal ImportService(TitanoDatabase db, ArchiveIngest ingest) + { + _db = db; + _ingest = ingest; + } + + /// The card with this identity, registered on first sight (the UI then asks for a nickname). + public async Task RegisterCardAsync(CardVolume volume, CancellationToken cancellationToken = default) + { + var now = DateTimeOffset.UtcNow; + if (await _db.MediaCards.FindByIdentityAsync(volume.Identity, cancellationToken).ConfigureAwait(false) is { } known) + { + await _db.MediaCards.TouchAsync(known.Id, now, cancellationToken).ConfigureAwait(false); + return known with { LastSeen = now }; + } + var card = new MediaCard + { + Id = Guid.CreateVersion7(), + Identity = volume.Identity, + VolumeSerial = volume.SerialNumber, + Label = volume.Label, + Capacity = volume.Capacity, + FirstSeen = now, + LastSeen = now, + }; + await _db.MediaCards.UpsertAsync(card, cancellationToken).ConfigureAwait(false); + return card; + } + + public static CardProfile ProfileOf(MediaCard card) => + card.ProfileJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.CardProfile) ?? new() : new(); + + /// Stores the nickname, defaults and auto-import choice of a card. + public async Task SaveCardAsync(Guid cardId, string? nickname, CardProfile profile, bool autoImport, CancellationToken cancellationToken = default) + { + var card = await _db.MediaCards.GetAsync(cardId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown card {cardId}.", nameof(cardId)); + var updated = card with + { + Nickname = string.IsNullOrWhiteSpace(nickname) ? null : nickname.Trim(), + ProfileJson = JsonSerializer.Serialize(profile, ArchiveJobJson.Default.CardProfile), + AutoImport = autoImport, + }; + await _db.MediaCards.UpsertAsync(updated, cancellationToken).ConfigureAwait(false); + return updated; + } + + /// Scans the source, reads light metadata and plans the import into the archive. Nothing is written + /// except the card's camera identity on its first plan. + /// Photos read so far. + public async Task PlanAsync(Archive archive, string sourceRoot, ImportOptions options, Guid? cardId, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var problems = new List(); + var files = CardScanner.Find(sourceRoot, options.IncludeSubfolders, cancellationToken); + var photos = CardScanner.ReadMetadata(sourceRoot, files, problems, progress, cancellationToken); + var card = cardId is { } id ? await _db.MediaCards.GetAsync(id, cancellationToken).ConfigureAwait(false) : null; + if (card is { CameraModel: null } && photos.Count > 0) + { + var first = photos[0].Metadata; + await _db.MediaCards.UpsertAsync(card with { CameraMake = first.Make, CameraModel = first.Model, CameraSerial = first.BodySerial }, + cancellationToken).ConfigureAwait(false); + } + var imported = card is null ? new HashSet() : await _db.MediaCards.GetImportedKeysAsync(card.Id, cancellationToken).ConfigureAwait(false); + var folders = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + var lookup = new IndexLookup(_db, archive.Id, paths, imported, folders); + return await ImportPlanner.PlanAsync(sourceRoot, paths.Root, photos, problems, options, lookup, cancellationToken).ConfigureAwait(false); + } + + /// Runs (or resumes) a confirmed plan; sees each batch after its history is recorded. + internal async Task ExecuteAsync(ImportJobParameters job, Guid jobId, int startAt, Func committed, + Func? progress, CancellationToken cancellationToken) + { + var archive = await _db.Archives.GetAsync(job.ArchiveId, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Unknown archive {job.ArchiveId}."); + ArchiveVolume.EnsureWritable(new ArchivePaths(archive.RootPath)); + var sink = new IngestSink(_ingest, archive, job.UserId?.ToString()); + return await new ImportExecutor(job.Plan, sink).RunAsync(startAt, job.Verify, async batch => + { + if (job.CardId is { } card) await RecordHistoryAsync(card, jobId, batch, cancellationToken).ConfigureAwait(false); + await committed(batch).ConfigureAwait(false); + }, progress, cancellationToken).ConfigureAwait(false); + } + + /// Dry run of "delete imported": every card file with a recorded import, checked against its archive copy. + public async Task PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress? progress = null, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var candidates = new List(); + foreach (var record in await _db.MediaCards.ListVerifiableImportsAsync(cardId, cancellationToken).ConfigureAwait(false)) + { + if (!File.Exists(Path.Combine(sourceRoot, record.Key.Path))) continue; + if (await _db.Frames.GetAsync(record.FrameId!.Value, cancellationToken).ConfigureAwait(false) is not { } frame) continue; + candidates.Add(new EraseCandidate(record.Key.Path, record.Key.Size, record.Sha256!, paths.FromRelative(frame.RelativePath))); + } + return await CardEraser.PlanAsync(sourceRoot, candidates, progress, cancellationToken).ConfigureAwait(false); + } + + private Task RecordHistoryAsync(Guid cardId, Guid jobId, ImportBatch batch, CancellationToken cancellationToken) + { + var now = DateTimeOffset.UtcNow; + var records = new List(); + foreach (var r in batch.Results) + { + var key = new ImportKey(r.Item.SourcePath, r.Item.Size, new DateTimeOffset(DateTime.SpecifyKind(r.Item.ModifiedUtc, DateTimeKind.Utc))); + switch (r.Status) + { + case ImportItemStatus.Copied or ImportItemStatus.Duplicate: + records.Add(new ImportRecord(cardId, key, r.Sha256, r.Commit?.FrameId, jobId, now)); + break; + case ImportItemStatus.Skipped when r.Item.Action == ImportAction.SkipInArchive: + records.Add(new ImportRecord(cardId, key, r.Item.Sha256, r.Item.ExistingFrameId, jobId, now)); + break; + } + } + return records.Count == 0 ? Task.CompletedTask : _db.MediaCards.RecordImportsAsync(records, cancellationToken); + } + + /// Planner questions answered from the index and the archive folder. + private sealed class IndexLookup(TitanoDatabase db, Guid archiveId, ArchivePaths paths, IReadOnlySet imported, HashSet indexedFolders) + : IImportLookup + { + public Task WasImportedAsync(CardFile file, CancellationToken cancellationToken) => + Task.FromResult(imported.Contains(new ImportKey(file.RelativePath, file.Size, new DateTimeOffset(DateTime.SpecifyKind(file.ModifiedUtc, DateTimeKind.Utc))))); + + public async Task> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) => + (await db.Frames.FindBySizeAndCaptureAsync(archiveId, size, capturedAt, cancellationToken).ConfigureAwait(false)) + .Select(f => new ArchiveMatch(f.Id, f.Sha256)).ToList(); + + public async Task FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) => + await db.Sequences.FindBySessionKeyAsync(archiveId, sessionKey, cancellationToken).ConfigureAwait(false) is { } id + ? (await db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder + : null; + + public bool IsFolderTaken(string folder) + { + if (indexedFolders.Contains(folder)) return true; + var path = paths.Sequence(folder).Folder; + return Directory.Exists(path) && Directory.EnumerateFileSystemEntries(path).Any(); + } + } + + /// Import destination "Archive": each plan session becomes (or continues) the sequence of its session key. + private sealed class IngestSink(ArchiveIngest ingest, Archive archive, string? createdBy) : IImportSink + { + public string StagingDirectory { get; } = new ArchivePaths(archive.RootPath).StagingDirectory; + + public async Task> CommitAsync(PlannedSession session, IReadOnlyList files, CancellationToken cancellationToken) + { + var frames = files.Select(f => new StagedFrame(f.StagedPath, Path.GetFileName(f.Item.SourcePath), f.Item.Size, f.Sha256, + MetadataReader.Read(f.StagedPath, cancellationToken), f.Item.CapturedAt, f.Item.FileName)).ToList(); + var results = await ingest.CommitSessionAsync(archive, new SequenceTarget(session.SessionKey, session.Title, session.Folder, createdBy), + frames, cancellationToken).ConfigureAwait(false); + return results.Select(r => new SinkCommit(r.FrameId, r.RelativePath, r.Duplicate)).ToList(); + } + } +} diff --git a/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs b/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs new file mode 100644 index 00000000..f0224e17 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ArchiveJobParameters.cs @@ -0,0 +1,38 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Import; +using Titano.Core.Model; + +namespace Titano.Data.Archiving.Jobs; + +/// : target = the archive. +public sealed record ScanParameters(bool Full); + +/// : target = the archive; null sequences = all of them. +public sealed record VerifyParameters(IReadOnlyList? SequenceIds); + +/// and : targets = sequences. +public sealed record ImageJobParameters(bool Force); + +/// : target = the archive. +public sealed record MaintenanceParameters(bool DryRun); + +/// : the exact plan the user confirmed. +public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId); + +/// A job ready to enqueue. +public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList Targets, string? Label, string? ParametersJson, JobPriority Priority); + +[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)] +[JsonSerializable(typeof(ScanParameters))] +[JsonSerializable(typeof(VerifyParameters))] +[JsonSerializable(typeof(ImageJobParameters))] +[JsonSerializable(typeof(MaintenanceParameters))] +[JsonSerializable(typeof(ImportJobParameters))] +[JsonSerializable(typeof(CardProfile))] +internal sealed partial class ArchiveJobJson : JsonSerializerContext +{ + /// Parameters of a job, or when it has none. + public static T Read(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo, T fallback) => + job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback; +} diff --git a/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs new file mode 100644 index 00000000..5d6af4b5 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ImageJobHandler.cs @@ -0,0 +1,51 @@ +using System.Globalization; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// +/// and over the target sequences. Progress counts +/// sequences; checkpoint: sequence:frame. +/// +internal sealed class ImageJobHandler(TitanoDatabase db, ArchiveImages images, JobKind kind) : IJobHandler +{ + public JobKind Kind => kind; + + public async Task RunAsync(JobContext context) + { + var force = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ImageJobParameters, new ImageJobParameters(false)).Force; + var (first, firstFrame) = Parse(context.Checkpoint); + var targets = context.Job.TargetIds; + for (var s = first; s < targets.Count; s++) + { + context.Cancellation.ThrowIfCancellationRequested(); + if (await db.Sequences.GetAsync(targets[s], context.Cancellation).ConfigureAwait(false) is not { } sequence + || await db.Archives.GetAsync(sequence.ArchiveId, context.Cancellation).ConfigureAwait(false) is not { } archive) + { + context.Log(Loc.L("Archive.Log.SequenceMissing", targets[s])); + continue; + } + var index = s; + ValueTask Checkpoint(int nextFrame, int frames) => + context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{index}:{nextFrame}"), index, targets.Count); + var start = s == first ? firstFrame : 0; + var result = kind == JobKind.Thumbs + ? await images.BuildThumbnailsAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false) + : await images.BuildProxiesAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false); + context.Log(Loc.L("Archive.Log.ImagesDone", sequence.DisplayTitle, result.Built, result.Reused, result.Undecodable)); + await context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{s + 1}:0"), s + 1, targets.Count).ConfigureAwait(false); + } + } + + private static (int Sequence, int Frame) Parse(string? checkpoint) + { + var parts = checkpoint?.Split(':'); + return parts is [var s, var f] + && int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence) + && int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame) + ? (sequence, frame) + : (0, 0); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs new file mode 100644 index 00000000..bf48eaa5 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ImportJobHandler.cs @@ -0,0 +1,33 @@ +using System.Globalization; +using System.Text.Json; +using Titano.Core.Import; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// +/// : executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the +/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion +/// continues there. +/// +internal sealed class ImportJobHandler(ImportService imports) : IJobHandler +{ + public JobKind Kind => JobKind.Import; + + public async Task RunAsync(JobContext context) + { + var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ImportJobParameters) : null) + ?? throw new InvalidOperationException($"Import job {context.Job.Id} has no plan."); + var start = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0; + var final = await imports.ExecuteAsync(parameters, context.Job.Id, start, async batch => + { + foreach (var failed in batch.Results.Where(r => r.Status == ImportItemStatus.Failed)) + context.Log(Loc.L("Archive.Log.ImportFailed", failed.Item.SourcePath, failed.Error)); + await context.CheckpointAsync(batch.NextItem.ToString(CultureInfo.InvariantCulture), batch.Progress.BytesDone, batch.Progress.BytesTotal) + .ConfigureAwait(false); + }, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false); + context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed)); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs b/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs new file mode 100644 index 00000000..23503433 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/MaintenanceJobHandler.cs @@ -0,0 +1,29 @@ +using System.Globalization; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// +/// : stale leases, leftover temporary files, thumbnail pack compaction, duplicate +/// flags, then an orphan count. Checkpoint: the number of steps done. +/// +internal sealed class MaintenanceJobHandler(TitanoDatabase db, ArchiveMaintenance maintenance) : IJobHandler +{ + public JobKind Kind => JobKind.Maintenance; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + var dryRun = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.MaintenanceParameters, new MaintenanceParameters(false)).DryRun; + var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var step) ? step : 0; + var report = await maintenance.RunAsync(archive, dryRun, first, + done => context.CheckpointAsync(done.ToString(CultureInfo.InvariantCulture), done, ArchiveMaintenance.StepCount), + context.Cancellation).ConfigureAwait(false); + if (dryRun) context.Log(Loc.L("Archive.Log.DryRun")); + context.Log(Loc.L("Archive.Log.MaintenanceDone", report.StaleLeases.Count, report.TemporaryFiles.Count, report.CompactedPacks.Count, + report.DuplicateFlagChanges)); + context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles)); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs new file mode 100644 index 00000000..3b010fd4 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/ScanJobHandler.cs @@ -0,0 +1,41 @@ +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// Resolves the archive a job targets (its first target id). +internal static class JobArchive +{ + public static async Task LoadAsync(TitanoDatabase db, Job job, CancellationToken cancellationToken) => + job.TargetIds.Count > 0 && await db.Archives.GetAsync(job.TargetIds[0], cancellationToken).ConfigureAwait(false) is { } archive + ? archive + : throw new InvalidOperationException($"{job.Kind} job {job.Id} does not target a known archive."); +} + +/// +/// (incremental, or full with ) and +/// . Checkpoint: after:<folder>, the last folder indexed. +/// +internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler +{ + private const string Prefix = "after:"; + + public JobKind Kind => kind; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + var mode = kind == JobKind.Rebuild ? ScanMode.Rebuild + : ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).Full ? ScanMode.Full + : ScanMode.Incremental; + var resume = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) ? checkpoint[Prefix.Length..] : null; + var summary = await indexer.ScanAsync(archive, mode, resume, async p => + { + if (p.Changed) await context.CheckpointAsync(Prefix + p.Folder, p.FoldersDone, p.FoldersTotal).ConfigureAwait(false); + else await context.ReportAsync(p.FoldersDone, p.FoldersTotal, p.Folder).ConfigureAwait(false); + }, context.Cancellation).ConfigureAwait(false); + foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error)); + context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed)); + } +} diff --git a/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs b/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs new file mode 100644 index 00000000..2dfa0180 --- /dev/null +++ b/src/Titano.Data/Archiving/Jobs/VerifyJobHandler.cs @@ -0,0 +1,47 @@ +using System.Globalization; +using Titano.Core.Archiving; +using Titano.Core.Localization; +using Titano.Core.Model; +using Titano.Data.Jobs; + +namespace Titano.Data.Archiving.Jobs; + +/// ; progress in bytes hashed. Checkpoint: sequence:frame. +internal sealed class VerifyJobHandler(TitanoDatabase db, ArchiveVerifier verifier) : IJobHandler +{ + /// Problems listed one per log line; the rest are counted (the index flags them all). + private const int LoggedProblems = 25; + + public JobKind Kind => JobKind.Verify; + + public async Task RunAsync(JobContext context) + { + var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false); + var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null)); + var report = await verifier.VerifyAsync(archive, parameters.SequenceIds, Parse(context.Checkpoint), + (position, done, total) => context.CheckpointAsync( + string.Create(CultureInfo.InvariantCulture, $"{position.Sequence}:{position.Frame}"), done, total), + (done, total) => context.ReportAsync(done, total), + context.Cancellation).ConfigureAwait(false); + foreach (var problem in report.Problems.Take(LoggedProblems)) + { + context.Log(problem.Health switch + { + FrameHealth.Missing => Loc.L("Archive.Log.FrameMissing", problem.RelativePath), + FrameHealth.Changed => Loc.L("Archive.Log.FrameChanged", problem.RelativePath), + _ => Loc.L("Archive.Log.FrameCorrupt", problem.RelativePath), + }); + } + context.Log(Loc.L("Archive.Log.VerifyDone", report.Checked, report.Ok, report.Hashed, report.Problems.Count)); + } + + private static VerifyPosition Parse(string? checkpoint) + { + var parts = checkpoint?.Split(':'); + return parts is [var s, var f] + && int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence) + && int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame) + ? new VerifyPosition(sequence, frame) + : default; + } +} diff --git a/src/Titano.Data/Archiving/MassRenamer.cs b/src/Titano.Data/Archiving/MassRenamer.cs new file mode 100644 index 00000000..9cc0af2e --- /dev/null +++ b/src/Titano.Data/Archiving/MassRenamer.cs @@ -0,0 +1,134 @@ +using Titano.Core.Archiving; +using Titano.Core.Diagnostics; +using Titano.Core.Localization; +using Titano.Core.Model; + +namespace Titano.Data.Archiving; + +/// The plan no longer matches the archive (a sequence moved or vanished since it was made). +public sealed class StaleRenamePlanException(Guid sequenceId) + : InvalidOperationException(Loc.L("Archive.Error.StalePlan")) +{ + public Guid SequenceId { get; } = sequenceId; +} + +/// +/// Mass rename of sequence folders and titles (BRIEF §7, §12.5): is the dry run; +/// makes exactly the planned changes to folders, sidecars and the index, step by step, with a +/// journal on the archive so an interrupted rename is completed by . +/// +public sealed class MassRenamer +{ + private readonly TitanoDatabase _db; + private readonly SequenceWriter _writer; + private readonly ArchiveIndexer _indexer; + + internal MassRenamer(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer) + { + _db = db; + _writer = writer; + _indexer = indexer; + } + + /// New folder template ({title} = the new title), or null to keep folders. + /// New title template ({title} = the current title), or null to keep titles. + public async Task PlanAsync(Archive archive, IReadOnlyList sequenceIds, string? folderTemplate, string? titleTemplate, + CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false); + var subjects = new List(sequenceIds.Count); + foreach (var id in sequenceIds) + { + if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id) + throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds)); + subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens)); + } + return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder), + folder => Directory.Exists(paths.Sequence(folder).Folder)); + } + + /// Applies a plan; each completed step is journaled before the next starts. + /// A sequence is no longer where the plan expects it; nothing of that step was changed. + public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress? progress = null, CancellationToken cancellationToken = default) => + RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken); + + /// Completes renames interrupted by a crash; returns how many plans were resumed. + public async Task ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default) + { + var paths = new ArchivePaths(archive.RootPath); + var pending = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList(); + foreach (var journal in pending) await RunAsync(archive, journal, resuming: true, null, cancellationToken).ConfigureAwait(false); + return pending.Count; + } + + /// Steps whose sequence the index no longer knows are dropped instead of failing the rename: + /// their sidecars still carry the ids, so the next scan indexes them wherever they ended up. + private async Task RunAsync(Archive archive, RenameJournal journal, bool resuming, IProgress? progress, CancellationToken cancellationToken) + { + var paths = new ArchivePaths(archive.RootPath); + ArchiveVolume.EnsureWritable(paths); + if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal)); + var completed = journal.Completed.ToHashSet(); + if (!resuming) + { + // A plan made before another change is refused before anything moves. + foreach (var step in journal.Plan.Steps) + { + if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder) + throw new StaleRenamePlanException(step.SequenceId); + } + } + RenameJournal.Write(paths, journal); + for (var i = 0; i < journal.Plan.Steps.Count; i++) + { + var step = journal.Plan.Steps[i]; + if (completed.Contains(step.SequenceId)) continue; + cancellationToken.ThrowIfCancellationRequested(); + try + { + await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false); + } + catch (StaleRenamePlanException) when (resuming) + { + Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there"); + } + completed.Add(step.SequenceId); + RenameJournal.Write(paths, journal with { Completed = [.. completed] }); + progress?.Report(i + 1); + } + RenameJournal.Delete(paths, journal.Plan.Id); + } + + /// Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it. + private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken) + { + await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false)) + { + var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId); + RenamePlanner.MoveFolder(paths, step); + var sequence = paths.Sequence(step.ToFolder); + var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId); + if (manifest.Title != step.ToTitle) + { + manifest = manifest with + { + Title = step.ToTitle, + Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + ManifestStore.WriteSequence(sequence.ManifestPath, manifest); + } + await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false)) + { + var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false); + await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false); + if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false); + await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false); + await tx.CommitAsync(cancellationToken).ConfigureAwait(false); + } + await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/Titano.Data/Archiving/SequenceMapping.cs b/src/Titano.Data/Archiving/SequenceMapping.cs new file mode 100644 index 00000000..7439a955 --- /dev/null +++ b/src/Titano.Data/Archiving/SequenceMapping.cs @@ -0,0 +1,106 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Model; + +namespace Titano.Data.Archiving; + +/// Sidecar records to index rows. Fields the sidecar does not carry are kept from the existing row. +internal static class SequenceMapping +{ + public static Frame ToFrame(SidecarFrame f, Guid sequenceId, SequencePaths sequence) => new() + { + Id = f.Id, + SequenceId = sequenceId, + Index = f.I, + RelativePath = sequence.ArchiveRelative(f.File), + FileName = Path.GetFileName(f.File), + OriginalFileName = f.Original, + Size = f.Size, + Sha256 = f.Sha256, + CapturedAt = f.CapturedAt, + UtcOffsetMinutes = f.UtcOffset, + TimestampSource = f.TimeSource, + ExposureTime = f.Exposure, + FNumber = f.FNumber, + Iso = f.Iso, + FocalLength = f.Focal, + ExposureBias = f.Bias, + WhiteBalance = f.Wb, + Orientation = f.Orientation, + Width = f.W, + Height = f.H, + Format = f.Format, + PreviewOffset = f.PreviewOffset, + PreviewLength = f.PreviewLength, + Flags = f.Flags, + }; + + public static List ToFrames(SequenceManifest manifest, SequencePaths sequence) => + manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList(); + + /// The current row; its render, analysis, sync and ownership fields are kept, and so are its + /// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only). + public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing) + { + var indexAhead = existing is not null && existing.SettingsVersion > m.Versions.Settings; + var times = m.Frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList(); + var flagged = m.Frames.Any(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0); + return new Sequence + { + Id = m.Id, + ArchiveId = archiveId, + Title = indexAhead ? existing!.Title : m.Title, + Folder = sequence.RelativeFolder, + CameraMake = m.Camera?.Make, + CameraModel = m.Camera?.Model, + CameraSerial = m.Camera?.Serial, + Lens = m.Lens, + FirstCapture = times.Count > 0 ? times[0] : null, + LastCapture = times.Count > 0 ? times[^1] : null, + FrameCount = m.Frames.Count, + MedianInterval = MedianInterval(times), + TotalBytes = m.Frames.Sum(f => f.Size), + Status = flagged ? m.Status | SequenceStatus.HasErrors : m.Status & ~SequenceStatus.HasErrors, + VerifiedAt = m.VerifiedAt, + Tags = indexAhead ? existing!.Tags : m.Tags, + Rating = indexAhead ? existing!.Rating : m.Rating, + Notes = indexAhead ? existing!.Notes : m.Notes, + SettingsVersion = indexAhead ? existing!.SettingsVersion : m.Versions.Settings, + FramesVersion = m.Versions.Frames, + UpdatedAt = m.UpdatedAt, + RenderedCount = existing?.RenderedCount ?? 0, + HeroFrameId = existing?.HeroFrameId, + SettingsJson = existing?.SettingsJson ?? ReadProject(sequence), + AnalysisJson = existing?.AnalysisJson, + Origin = existing?.Origin ?? SequenceOrigin.Local, + SyncState = existing?.SyncState ?? SyncState.LocalOnly, + OwnerUserId = existing?.OwnerUserId ?? (Guid.TryParse(m.CreatedBy, out var owner) ? owner : null), + WarningCount = existing?.WarningCount ?? 0, + }; + } + + private static double? MedianInterval(List times) + { + if (times.Count < 2) return null; + var gaps = new double[times.Count - 1]; + for (var i = 1; i < times.Count; i++) gaps[i - 1] = (times[i] - times[i - 1]).TotalSeconds; + Array.Sort(gaps); + var mid = gaps.Length / 2; + return gaps.Length % 2 == 1 ? gaps[mid] : (gaps[mid - 1] + gaps[mid]) / 2; + } + + /// The project file is owned by the editor; a rebuilt index takes its text as-is when it is valid JSON. + private static string? ReadProject(SequencePaths sequence) + { + if (!File.Exists(sequence.ProjectPath)) return null; + var text = File.ReadAllText(sequence.ProjectPath); + try + { + using (JsonDocument.Parse(text)) return text; + } + catch (JsonException) + { + return null; + } + } +} diff --git a/src/Titano.Data/Archiving/SequenceWriter.cs b/src/Titano.Data/Archiving/SequenceWriter.cs new file mode 100644 index 00000000..fe636cf4 --- /dev/null +++ b/src/Titano.Data/Archiving/SequenceWriter.cs @@ -0,0 +1,92 @@ +using System.Collections.Concurrent; +using Titano.Core.Archiving; +using Titano.Core.Imaging; + +namespace Titano.Data.Archiving; + +/// Settings shared by the archive services. +public sealed record ArchiveServiceOptions +{ + /// Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows). + public ImageDecoder Decoder { get; init; } = new([]); + + /// Name written into lease files so a waiting writer can say who holds a sequence. + public string Owner { get; init; } = "titano@" + Environment.MachineName; + + /// How long a writer waits for a sequence held by another process before giving up. + public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2); + + public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration; +} + +/// +/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease +/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar, +/// with an interrupted ingest batch completed first. +/// +internal sealed class SequenceWriter(ArchiveServiceOptions options) +{ + private readonly ConcurrentDictionary _gates = new(); + + public ArchiveServiceOptions Options => options; + + /// Holds the sequence exclusively until disposed. + public async Task LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken) + { + var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1)); + await gate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait, + options.LeaseDuration, cancellationToken).ConfigureAwait(false); + return new Held(gate, lease); + } + catch + { + gate.Release(); + throw; + } + } + + /// + /// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first. + /// + /// The sidecar in the folder belongs to another sequence. + public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId) + { + var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); + if (manifest.Id != sequenceId) + throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}."); + if (IngestJournal.Recover(sequence, manifest) is { } recovered) + { + ManifestStore.WriteSequence(sequence.ManifestPath, recovered); + manifest = recovered; + } + IngestJournal.Delete(sequence); + return manifest; + } + + /// Applies to the sidecar under the lock; a null result writes nothing. + public async Task UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder, + Func edit, CancellationToken cancellationToken) + { + var sequence = archive.Sequence(folder); + await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false)) + { + var manifest = ReadLocked(sequence, sequenceId); + if (edit(manifest) is not { } updated) return manifest; + ManifestStore.WriteSequence(sequence.ManifestPath, updated); + return updated; + } + } + + private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable + { + public ValueTask DisposeAsync() + { + lease.Dispose(); + gate.Release(); + return ValueTask.CompletedTask; + } + } +} diff --git a/src/Titano.Data/Migrations/postgres/0002_archive_index.sql b/src/Titano.Data/Migrations/postgres/0002_archive_index.sql new file mode 100644 index 00000000..b872e6d2 --- /dev/null +++ b/src/Titano.Data/Migrations/postgres/0002_archive_index.sql @@ -0,0 +1,23 @@ +-- Archive index state (BRIEF §7, §14.4). Mirrors sqlite/0002_archive_index.sql. + +-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it. +CREATE TABLE sequence_sessions ( + archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + session_key text NOT NULL, + sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + PRIMARY KEY (archive_id, session_key) +); +CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id); + +-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time. +-- Times are UTC ticks (100 ns), compared exactly with the file system's. +CREATE TABLE archive_folders ( + archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + folder text COLLATE "C" NOT NULL, + sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL, + sidecar_size bigint, + sidecar_ticks bigint, + photos_ticks bigint, + scanned_at timestamptz NOT NULL, + PRIMARY KEY (archive_id, folder) +); diff --git a/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql b/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql new file mode 100644 index 00000000..bace37eb --- /dev/null +++ b/src/Titano.Data/Migrations/sqlite/0002_archive_index.sql @@ -0,0 +1,23 @@ +-- Archive index state (BRIEF §7, §14.4). Mirrors postgres/0002_archive_index.sql. + +-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it. +CREATE TABLE sequence_sessions ( + archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + session_key TEXT NOT NULL, + sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE, + PRIMARY KEY (archive_id, session_key) +); +CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id); + +-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time. +-- Times are UTC ticks (100 ns), compared exactly with the file system's. +CREATE TABLE archive_folders ( + archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE, + folder TEXT NOT NULL, + sequence_id TEXT REFERENCES sequences (id) ON DELETE SET NULL, + sidecar_size INTEGER, + sidecar_ticks INTEGER, + photos_ticks INTEGER, + scanned_at TEXT NOT NULL, + PRIMARY KEY (archive_id, folder) +); diff --git a/src/Titano.Data/Repositories/ArchiveRepository.cs b/src/Titano.Data/Repositories/ArchiveRepository.cs index 717eb4f9..dec57a0a 100644 --- a/src/Titano.Data/Repositories/ArchiveRepository.cs +++ b/src/Titano.Data/Repositories/ArchiveRepository.cs @@ -1,7 +1,11 @@ +using Titano.Core.Archiving; using Titano.Core.Model; namespace Titano.Data.Repositories; +/// What the last scan saw in one sequence folder (sidecar and photo-directory stamps). +public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? Sidecar, FileStamp? Photos); + /// Archive roots (BRIEF §5, §7). public sealed class ArchiveRepository { @@ -10,6 +14,9 @@ public sealed class ArchiveRepository private static readonly string UpsertSql = SqlDialect.Upsert("archives", ["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1); + private static readonly string FolderStateUpsertSql = SqlDialect.Upsert("archive_folders", + ["archive_id", "folder", "sequence_id", "sidecar_size", "sidecar_ticks", "photos_ticks", "scanned_at"], 2); + private readonly DbAccess _db; internal ArchiveRepository(DbAccess db) => _db = db; @@ -29,6 +36,31 @@ public sealed class ArchiveRepository public async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) => await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0; + /// Scan stamps of every folder of an archive, by folder. + public async Task> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) + { + var rows = await _db.ListAsync( + "SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId], + r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1), + r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null, + r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null), + cancellationToken).ConfigureAwait(false); + return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal); + } + + public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync(FolderStateUpsertSql, + [archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow], + cancellationToken).ConfigureAwait(false); + + public async Task DeleteFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) => + await _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1 AND folder = $2", [archiveId, folder], cancellationToken) + .ConfigureAwait(false) > 0; + + /// Forgets every scan stamp of an archive, so the next scan reads every sidecar (rebuild). + public Task ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken); + private static Archive Map(DbRow r) => new() { Id = r.GetGuid(0), diff --git a/src/Titano.Data/Repositories/FrameRepository.cs b/src/Titano.Data/Repositories/FrameRepository.cs index 30350e5e..ae064902 100644 --- a/src/Titano.Data/Repositories/FrameRepository.cs +++ b/src/Titano.Data/Repositories/FrameRepository.cs @@ -4,6 +4,15 @@ using Titano.Core.Model; namespace Titano.Data.Repositories; +/// Integrity fields a verify rewrites: the flags and the hash of a newly hashed file. +public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256); + +/// A frame and the sequence it belongs to. +public readonly record struct FrameRef(Guid FrameId, Guid SequenceId); + +/// A frame whose content (SHA-256) also exists elsewhere in the archive. +public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath); + /// Light statistics of one frame, written by the analysis job. public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh); @@ -25,6 +34,12 @@ public sealed class FrameRepository /// Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row). private const int Batch = 1000; + /// Rows per multi-row upsert: 30 x 27 parameters stays below SQLite's historical 999-variable limit. + private const int UpsertRows = 30; + + /// Light statistics belong to the analysis job; a sync from the sidecar never clears them. + private static readonly string[] AnalysisColumns = ["mean_log_luminance", "clip_low", "clip_high"]; + private readonly DbAccess _db; internal FrameRepository(DbAccess db) => _db = db; @@ -112,6 +127,89 @@ public sealed class FrameRepository return updated; }, cancellationToken); + /// + /// Makes the frames of a sequence exactly (from its sidecar): rows are upserted by id, + /// rows not listed are deleted, light statistics of kept rows survive. Indexes may be renumbered freely. + /// + public Task SyncSequenceAsync(Guid sequenceId, IReadOnlyList frames, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + // Park current rows on negative indexes so renumbered rows never collide on (sequence_id, idx). + await session.ExecuteAsync("UPDATE frames SET idx = -idx - 1 WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); + var updates = string.Join(", ", ColumnNames.Skip(1).Where(c => !AnalysisColumns.Contains(c)).Select(c => $"{c} = excluded.{c}")); + for (var start = 0; start < frames.Count; start += UpsertRows) + { + var count = Math.Min(UpsertRows, frames.Count - start); + var sql = new StringBuilder("INSERT INTO frames (").Append(Columns).Append(") VALUES "); + var args = new DbParam[count * ColumnNames.Length]; + for (var i = 0; i < count; i++) + { + var values = Values(frames[start + i]); + sql.Append(i > 0 ? ", (" : "("); + for (var c = 0; c < values.Length; c++) + { + args[i * values.Length + c] = new DbParam(values[c]); + sql.Append(c > 0 ? ", $" : "$").Append(i * values.Length + c + 1); + } + sql.Append(')'); + } + sql.Append(" ON CONFLICT (id) DO UPDATE SET ").Append(updates); + await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false); + } + return await session.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1 AND idx < 0", [sequenceId], cancellationToken).ConfigureAwait(false); + }, cancellationToken); + + /// Number of frames of a sequence. + public async Task CountBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) => + (int)await _db.ScalarAsync("SELECT count(*) FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false); + + /// Frames of an archive with this size and capture time (card import duplicate candidates). + public Task> FindBySizeAndCaptureAsync(Guid archiveId, long size, DateTime capturedAt, CancellationToken cancellationToken = default) => + _db.ListAsync( + $"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + "WHERE s.archive_id = $1 AND f.size = $2 AND f.captured_at = $3", + [archiveId, size, SequenceRepository.Wall(capturedAt)], Map, cancellationToken); + + /// Every frame of an archive whose SHA-256 occurs more than once, grouped by hash then sequence and index. + public Task> ListDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT f.sha256, f.id, f.sequence_id, f.idx, f.relative_path FROM frames f JOIN sequences s ON s.id = f.sequence_id " + + "WHERE s.archive_id = $1 AND f.sha256 IN (SELECT f2.sha256 FROM frames f2 JOIN sequences s2 ON s2.id = f2.sequence_id " + + "WHERE s2.archive_id = $1 AND f2.sha256 IS NOT NULL GROUP BY f2.sha256 HAVING count(*) > 1) " + + "ORDER BY f.sha256, s.first_capture, f.sequence_id, f.idx", + [archiveId], r => new DuplicateFrame(r.GetString(0), r.GetGuid(1), r.GetGuid(2), r.GetInt32(3), r.GetString(4)), cancellationToken); + + /// Frames of an archive that carry every bit of . + public Task> ListFlaggedAsync(Guid archiveId, FrameFlags flags, CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT f.id, f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 AND (f.flags & $2) = $2", + [archiveId, (int)flags], r => new FrameRef(r.GetGuid(0), r.GetGuid(1)), cancellationToken); + + /// Rewrites the relative paths of a sequence's frames after its folder moved (mass rename). + /// Only paths still under change, so repeating it after a crash is harmless. + public Task RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default) + { + // substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder. + var prefix = oldFolder + "/"; + var length = prefix.EnumerateRunes().Count(); + return _db.ExecuteAsync( + "UPDATE frames SET relative_path = $2 || substr(relative_path, $3) WHERE sequence_id = $1 AND substr(relative_path, 1, $4) = $5", + [sequenceId, newFolder, length, length, prefix], cancellationToken); + } + + /// Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated. + public Task UpdateIntegrityAsync(IReadOnlyList frames, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var updated = 0; + foreach (var f in frames) + { + updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1", + [f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false); + } + return updated; + }, cancellationToken); + /// Removes every frame of a sequence; returns how many. public Task DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) => _db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken); diff --git a/src/Titano.Data/Repositories/MediaCardRepository.cs b/src/Titano.Data/Repositories/MediaCardRepository.cs index ab95cc5f..d42c6971 100644 --- a/src/Titano.Data/Repositories/MediaCardRepository.cs +++ b/src/Titano.Data/Repositories/MediaCardRepository.cs @@ -96,6 +96,16 @@ public sealed class MediaCardRepository await _db.ScalarAsync("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4", [cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null; + /// Every recorded import from a card that has a hash and a frame (candidates for a verified card erase). + public Task> ListVerifiableImportsAsync(Guid cardId, CancellationToken cancellationToken = default) => + _db.ListAsync( + "SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history " + + "WHERE card_id = $1 AND sha256 IS NOT NULL AND frame_id IS NOT NULL ORDER BY path", + [cardId], + r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)), + r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)), + cancellationToken); + /// Most recent imports from a card. public Task> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) => _db.ListAsync( diff --git a/src/Titano.Data/Repositories/SequenceRepository.cs b/src/Titano.Data/Repositories/SequenceRepository.cs index 4912cec5..eb566ded 100644 --- a/src/Titano.Data/Repositories/SequenceRepository.cs +++ b/src/Titano.Data/Repositories/SequenceRepository.cs @@ -79,6 +79,9 @@ public sealed record ArchiveHealth( long CorruptFrames, long DuplicateFrames); +/// Where a sequence lives in its archive. +public sealed record SequenceLocation(Guid Id, string Folder, string? Title); + /// Sequences (projects) with their tags; Library listing and Archive statistics. public sealed class SequenceRepository { @@ -259,6 +262,45 @@ public sealed class SequenceRepository }, cancellationToken); } + /// Folder and title of every sequence of an archive, by folder. + public Task> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) => + _db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId], + r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken); + + /// The sequence created for a session key (automatic grouping of uploads and card sessions). + public Task FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) => + _db.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", + [archiveId, sessionKey], cancellationToken); + + /// Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to. + public Task MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + await session.ExecuteAsync( + "INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING", + [archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false); + return (await session.ScalarAsync("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2", + [archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value; + }, cancellationToken); + + /// Moves a sequence to another folder and/or title (mass rename); false when it does not exist. + public Task RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) => + _db.InTransactionAsync(async session => + { + var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken) + .ConfigureAwait(false); + if (current is null) return false; + var titleChanged = current.Title != title; + await UpsertRowAsync(session, current with + { + Folder = folder, + Title = title, + SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion, + UpdatedAt = DateTimeOffset.UtcNow, + }, cancellationToken).ConfigureAwait(false); + return true; + }, cancellationToken); + private static List Filter(SequenceQuery query, SqlParameters p) { var where = new List(); diff --git a/tests/Titano.Core.Tests/Archiving/ArchiveFolderTests.cs b/tests/Titano.Core.Tests/Archiving/ArchiveFolderTests.cs new file mode 100644 index 00000000..78748418 --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/ArchiveFolderTests.cs @@ -0,0 +1,182 @@ +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveFolderTests(TestContext context) +{ + private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); + + private ArchivePaths NewArchive() => new(context.CreateTempDirectory("archive")); + + [Test] + public void SequenceFoldersAreFoundInLayoutAndAdoptedForms() + { + var archive = NewArchive(); + TestPhotos.WriteSeries(Path.Combine(archive.Root, "2026", "2026-09-28 Canon EOS R5", "originals"), Start, 3, TimeSpan.FromSeconds(10)); + TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour"), Start, 2, TimeSpan.FromSeconds(10)); + TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour", "selects"), Start, 1, TimeSpan.FromSeconds(10)); + TestPhotos.WriteSeries(Path.Combine(archive.Root, ".hidden"), Start, 1, TimeSpan.FromSeconds(10)); + File.WriteAllBytes(Path.Combine(archive.Root, "Trips", "._IMG_0001.JPG"), [1]); + Directory.CreateDirectory(Path.Combine(archive.Root, "empty")); + + var folders = ArchiveScanner.FindSequenceFolders(archive); + Assert.SequenceEqual(["2026/2026-09-28 Canon EOS R5", "Trips/Harbour", "Trips/Harbour/selects"], folders.Select(f => f.RelativeFolder)); + Assert.True(folders.All(f => f.Sidecar is null && f.Photos is not null)); + Assert.SequenceEqual(["originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG"], + ArchiveScanner.ListPhotos(archive.Sequence("2026/2026-09-28 Canon EOS R5"))); + } + + [Test] + public void AdoptionBuildsASidecarInCaptureOrderWithoutTouchingPhotos() + { + var archive = NewArchive(); + var folder = Path.Combine(archive.Root, "Harbour sunset"); + // File names run against capture order: the sidecar follows capture time. + TestPhotos.Write(Path.Combine(folder, "a.jpg"), Start.AddSeconds(20), 1); + TestPhotos.Write(Path.Combine(folder, "b.jpg"), Start, 2); + TestPhotos.Write(Path.Combine(folder, "c.jpg"), Start.AddSeconds(10), 3); + var before = Directory.GetFiles(folder).ToDictionary(f => f, File.ReadAllBytes); + + var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), archive.Sequence("Harbour sunset"), "test"); + Assert.SequenceEqual(["b.jpg", "c.jpg", "a.jpg"], manifest.Frames.Select(f => f.File)); + Assert.SequenceEqual([0, 1, 2], manifest.Frames.Select(f => f.I)); + Assert.Equal("Harbour sunset", manifest.Title); + Assert.Equal("Canon EOS R5", manifest.Camera!.Model); + Assert.True(manifest.Frames.All(f => f.Sha256 is null && f.Size > 0 && f.Format == ImageFileFormat.Jpeg && f.W == 48)); + foreach (var (path, bytes) in before) Assert.BytesEqual(bytes, File.ReadAllBytes(path)); + } + + [Test] + public void ReconcileIndexesNewPhotosAndFlagsMissingOnes() + { + var archive = NewArchive(); + var sequence = archive.Sequence("s"); + var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10)); + var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null); + Assert.Null(SequenceAdopter.Reconcile(sequence, manifest)); + + File.Delete(files[1]); + TestPhotos.Write(Path.Combine(sequence.Originals, "IMG_0000.JPG"), Start.AddSeconds(-10), 9); + var orphans = SequenceAdopter.FindOrphans(sequence, manifest); + Assert.SequenceEqual(["originals/IMG_0000.JPG"], orphans.UnindexedFiles); + Assert.SequenceEqual(["originals/IMG_0002.JPG"], orphans.MissingFiles.Select(f => f.File)); + + var reconciled = SequenceAdopter.Reconcile(sequence, manifest)!; + Assert.SequenceEqual(["originals/IMG_0000.JPG", "originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG", "originals/IMG_0004.JPG"], + reconciled.Frames.Select(f => f.File)); + Assert.Equal(FrameFlags.Missing, reconciled.Frames[2].Flags); + Assert.Equal(manifest.Frames[0].Id, reconciled.Frames[1].Id); + + TestPhotos.Write(files[1], Start.AddSeconds(10), 2); + var restored = SequenceAdopter.Reconcile(sequence, reconciled)!; + Assert.Equal(FrameFlags.None, restored.Frames[2].Flags); + } + + [Test] + public async Task VerifyDetectsAFlippedByteAMissingFileAndAChangedFile() + { + var archive = NewArchive(); + var sequence = archive.Sequence("s"); + var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10)); + var adopted = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null); + var hashed = new List(); + foreach (var frame in adopted.Frames) + { + var check = await FrameVerifier.CheckAsync(sequence, frame); + Assert.Equal(FrameHealth.Hashed, check.Health); + hashed.Add(check.Updated); + } + + var bytes = File.ReadAllBytes(files[0]); + bytes[bytes.Length / 2] ^= 0x01; + File.WriteAllBytes(files[0], bytes); + File.Delete(files[1]); + File.AppendAllText(files[2], "edit"); + + var health = new List(); + foreach (var frame in hashed) health.Add((await FrameVerifier.CheckAsync(sequence, frame)).Health); + Assert.SequenceEqual([FrameHealth.Corrupt, FrameHealth.Missing, FrameHealth.Changed, FrameHealth.Ok], health); + var corrupt = await FrameVerifier.CheckAsync(sequence, hashed[0]); + Assert.Equal(FrameFlags.Corrupt, corrupt.Flags); + Assert.Equal(hashed[0].Sha256, corrupt.Updated.Sha256); + } + + [Test] + public void TheIndexSnapshotIsUsedOnlyForUnchangedSidecars() + { + var archive = NewArchive(); + Directory.CreateDirectory(archive.MetaDirectory); + var stampA = new FileStamp(10, new DateTime(2026, 9, 1, 0, 0, 0, DateTimeKind.Utc)); + using (var writer = new IndexSnapshot.Writer(archive.SnapshotPath)) + { + writer.Add("a", stampA, "{\"a\":1}"u8); + writer.Add("b/ç", new FileStamp(20, stampA.ModifiedUtc), "{\"b\":2}"u8); + writer.Commit(); + } + var snapshot = IndexSnapshot.TryLoad(archive.SnapshotPath)!; + Assert.Equal(2, snapshot.Count); + Assert.BytesEqual("{\"a\":1}"u8, snapshot.TryGet("a", stampA)); + Assert.Null(snapshot.TryGet("a", stampA with { Size = 11 })); + Assert.NotNull(snapshot.TryGet("b/ç", new FileStamp(20, stampA.ModifiedUtc))); + + var bytes = File.ReadAllBytes(archive.SnapshotPath); + bytes[12] ^= 0x40; + File.WriteAllBytes(archive.SnapshotPath, bytes); + Assert.Null(IndexSnapshot.TryLoad(archive.SnapshotPath)); + } + + [Test] + public void RenamePlansAvoidTakenFoldersAndMoveIdempotently() + { + var archive = NewArchive(); + Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old a")); + Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old b")); + Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "2026-09-28 Sunset")); + var a = new RenameSubject(Guid.NewGuid(), "2026/old a", null, Start, "Canon", "Canon EOS R5", null); + var b = new RenameSubject(Guid.NewGuid(), "2026/old b", "Dawn", Start, "Canon", "Canon EOS R5", null); + var plan = RenamePlanner.Plan(Guid.NewGuid(), [a, b], "{year}/{date} {title}", "Sunset", + ["2026/old a", "2026/old b"], f => Directory.Exists(archive.FromRelative(f))); + Assert.SequenceEqual(["2026/2026-09-28 Sunset (2)", "2026/2026-09-28 Sunset (3)"], plan.Steps.Select(s => s.ToFolder)); + Assert.True(plan.Steps.All(s => s.ToTitle == "Sunset")); + Assert.Equal("Dawn", plan.Steps[1].FromTitle); + + RenamePlanner.MoveFolder(archive, plan.Steps[0]); + RenamePlanner.MoveFolder(archive, plan.Steps[0]); + Assert.False(Directory.Exists(archive.FromRelative("2026/old a"))); + Assert.True(Directory.Exists(archive.FromRelative("2026/2026-09-28 Sunset (2)"))); + + var titleOnly = RenamePlanner.Plan(Guid.NewGuid(), [b], null, "{title} morning", ["2026/old b"], _ => false); + Assert.Equal("2026/old b", titleOnly.Steps.Single().ToFolder); + Assert.Equal("Dawn morning", titleOnly.Steps.Single().ToTitle); + Assert.Empty(RenamePlanner.Plan(Guid.NewGuid(), [b], "2026/old b", null, ["2026/old b"], _ => true).Steps); + } + + [Test] + public void ThumbnailsAndProxiesAreRenderedFromTheFrame() + { + var archive = NewArchive(); + var sequence = archive.Sequence("s"); + Directory.CreateDirectory(sequence.Originals); + File.WriteAllBytes(Path.Combine(sequence.Originals, "big.jpg"), TestPhotos.Jpeg(Start, 7, width: 640, height: 480, orientation: 6)); + var frame = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null).Frames.Single(); + var renderer = new FrameImageRenderer(new ImageDecoder([])); + + var thumb = renderer.Thumbnail(sequence, frame)!; + Assert.Equal((240, 320), (thumb.Width, thumb.Height)); + Assert.Equal(DecodeTier.Native, thumb.Tier); + Assert.Equal(240, JpegDecoder.ReadInfo(thumb.Jpeg).Width); + + var proxy = renderer.Proxy(sequence, frame, new SidecarCamera { Make = "Canon", Model = "Canon EOS R5" }, new ProxyPolicy { LongEdge = 400 })!; + Assert.Equal((300, 400), (proxy.Width, proxy.Height)); + var exif = MetadataReader.Read(new MemoryStream(proxy.Jpeg), "p.jpg"); + Assert.Equal(Start, exif.CapturedAt); + Assert.Equal("Canon EOS R5", exif.Model); + Assert.Equal(1, exif.Orientation); + } +} diff --git a/tests/Titano.Core.Tests/Archiving/LeaseFileTests.cs b/tests/Titano.Core.Tests/Archiving/LeaseFileTests.cs new file mode 100644 index 00000000..7b8e98b8 --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/LeaseFileTests.cs @@ -0,0 +1,113 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class LeaseFileTests(TestContext context) +{ + private static void WriteRecord(string path, LeaseRecord record) => + File.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord)); + + [Test] + public void ALeaseIsExclusiveUntilReleased() + { + var locks = context.CreateTempDirectory(); + using (var a = LeaseFile.TryAcquire(locks, "seq-1", "server")) + { + Assert.NotNull(a); + Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client")); + using var other = LeaseFile.TryAcquire(locks, "seq-2", "client"); + Assert.NotNull(other); + } + using var b = LeaseFile.TryAcquire(locks, "seq-1", "client"); + Assert.NotNull(b); + Assert.Equal("client", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner); + } + + [Test] + public void AnExpiredLeaseOfACrashedHolderIsTakenOver() + { + var locks = context.CreateTempDirectory(); + var past = DateTimeOffset.UtcNow.AddMinutes(-10); + WriteRecord(LeaseFile.PathFor(locks, "seq-1"), new LeaseRecord("dead", "crashed", "pc", 1, past, past.AddMinutes(2))); + using var lease = LeaseFile.TryAcquire(locks, "seq-1", "survivor"); + Assert.NotNull(lease); + Assert.Equal("survivor", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner); + Assert.Count(1, Directory.GetFiles(locks)); + } + + [Test] + public async Task AHeldLeaseIsRenewedPastItsDuration() + { + var locks = context.CreateTempDirectory(); + using var lease = LeaseFile.TryAcquire(locks, "seq-1", "server", TimeSpan.FromMilliseconds(300)); + Assert.NotNull(lease); + await Task.Delay(900); + Assert.True(lease.IsHeld); + Assert.Greater((LeaseFile.Read(lease.Path)!.Expires - DateTimeOffset.UtcNow).TotalMilliseconds, 0); + Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client", TimeSpan.FromMilliseconds(300))); + } + + [Test] + public void AFreshEmptyLeaseFileIsNotStaleButAnOldOneIs() + { + var locks = context.CreateTempDirectory(); + var path = LeaseFile.PathFor(locks, "seq-1"); + File.WriteAllBytes(path, []); + Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client")); + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddHours(-1)); + using var lease = LeaseFile.TryAcquire(locks, "seq-1", "client"); + Assert.NotNull(lease); + } + + [Test] + public void ALostLeaseIsDetectedAndNotDeletedByItsFormerHolder() + { + var locks = context.CreateTempDirectory(); + var lease = LeaseFile.TryAcquire(locks, "seq-1", "slow"); + Assert.NotNull(lease); + var now = DateTimeOffset.UtcNow; + WriteRecord(lease.Path, new LeaseRecord("thief", "other", "pc", 2, now, now.AddMinutes(2))); + Assert.False(lease.Renew()); + Assert.False(lease.IsHeld); + lease.Dispose(); + Assert.Equal("thief", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Token); + } + + [Test] + public async Task WaitingForABusyLeaseTimesOutWithItsHolder() + { + var locks = context.CreateTempDirectory(); + using var held = LeaseFile.TryAcquire(locks, "seq-1", "server"); + var error = await Assert.ThrowsAsync(() => + LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromMilliseconds(200))); + Assert.Equal("server", error.Holder!.Owner); + } + + [Test] + public async Task WaitingForALeaseSucceedsOnceItIsReleased() + { + var locks = context.CreateTempDirectory(); + var held = LeaseFile.TryAcquire(locks, "seq-1", "server")!; + var waiting = LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromSeconds(10)); + await Task.Delay(100); + held.Dispose(); + using var lease = await waiting; + Assert.Equal("client", lease.Record.Owner); + } + + [Test] + public void CleanupRemovesOnlyStaleLeasesAndItsDryRunNothing() + { + var locks = context.CreateTempDirectory(); + var past = DateTimeOffset.UtcNow.AddMinutes(-10); + WriteRecord(LeaseFile.PathFor(locks, "dead"), new LeaseRecord("t", "crashed", "pc", 1, past, past.AddMinutes(1))); + using var live = LeaseFile.TryAcquire(locks, "live", "server"); + Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: true)); + Assert.Count(2, Directory.GetFiles(locks)); + Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: false)); + Assert.SequenceEqual([LeaseFile.PathFor(locks, "live")], Directory.GetFiles(locks)); + } +} diff --git a/tests/Titano.Core.Tests/Archiving/ManifestStoreTests.cs b/tests/Titano.Core.Tests/Archiving/ManifestStoreTests.cs new file mode 100644 index 00000000..4c81061a --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/ManifestStoreTests.cs @@ -0,0 +1,157 @@ +using System.Text; +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Imaging; +using Titano.Core.IO; +using Titano.Core.Model; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class ManifestStoreTests(TestContext context) +{ + private static SequenceManifest Sample() => new() + { + Id = Guid.CreateVersion7(), + Title = "Harbour sunset", + Camera = new SidecarCamera { Make = "Canon", Model = "Canon EOS R5", Serial = "0431" }, + Lens = "RF24-70mm F2.8 L IS USM", + Frames = + [ + new SidecarFrame + { + Id = Guid.CreateVersion7(), I = 0, File = "originals/2026-09-28_18-02-11_0001.CR2", Original = "IMG_0001.CR2", Size = 25_000_000, + Sha256 = new string('a', 64), CapturedAt = new DateTime(2026, 9, 28, 18, 2, 11, 250), UtcOffset = 120, + TimeSource = TimestampSource.ExifOriginal, Exposure = 1.0 / 250, FNumber = 8, Iso = 100, Focal = 24, Bias = -0.3, Wb = 0, + Orientation = 6, W = 8192, H = 5464, Format = ImageFileFormat.Cr2, PreviewOffset = 1234, PreviewLength = 567_890, + Flags = FrameFlags.Excluded, + }, + ], + Tags = ["sunset", "harbour"], + Rating = 4, + Notes = "Windy", + Status = SequenceStatus.ThumbsBuilt | SequenceStatus.Verified, + Versions = new ManifestVersions { Settings = 3, Frames = 7 }, + CreatedBy = "alberto", + Created = new DateTimeOffset(2026, 9, 28, 20, 0, 0, TimeSpan.Zero), + UpdatedAt = new DateTimeOffset(2026, 9, 29, 8, 0, 0, TimeSpan.Zero), + VerifiedAt = new DateTimeOffset(2026, 9, 30, 8, 0, 0, TimeSpan.Zero), + SessionKeys = ["card-1234abcd-20260928-180211"], + }; + + private string SidecarPath() => Path.Combine(context.CreateTempDirectory(), ".titano", "sequence.json"); + + private static string Canonical(SequenceManifest m) => JsonSerializer.Serialize(m, ArchiveJson.Default.SequenceManifest); + + [Test] + public void SequenceSidecarRoundTripsEveryField() + { + var path = SidecarPath(); + var original = Sample(); + ManifestStore.WriteSequence(path, original); + var read = ManifestStore.ReadSequence(path); + Assert.Equal(Canonical(original), Canonical(read)); + Assert.Equal(original.Frames[0], read.Frames[0] with { Extra = original.Frames[0].Extra }); + var json = File.ReadAllText(path); + foreach (var field in new[] { "\"i\":", "\"file\":", "\"sha256\":", "\"capturedAt\":", "\"fNumber\":", "\"w\":", "\"previewOffset\":", "\"flags\":" }) + Assert.Contains(field, json); + } + + [Test] + public void UnknownFieldsAndEnumValuesFromANewerBuildAreReadAndKept() + { + var path = SidecarPath(); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var id = Guid.CreateVersion7(); + var frameId = Guid.CreateVersion7(); + File.WriteAllText(path, $$""" + { + "schemaVersion": 1, "id": "{{id}}", "title": "Comet", "futureBlock": { "a": [1, 2] }, + "camera": { "make": "Nikon", "sensorTemp": 31 }, + "frames": [ { "id": "{{frameId}}", "i": 0, "file": "x.nef", "size": 10, "format": "Nef2030", "timeSource": "Atomic", + "flags": 4096, "aiScore": 0.9 } ], + "status": 1024, "versions": { "settings": 1, "frames": 2, "renders": 5 }, + } + """); + var read = ManifestStore.ReadSequence(path); + Assert.Equal(id, read.Id); + Assert.Equal(ImageFileFormat.Unknown, read.Frames[0].Format); + Assert.Equal(TimestampSource.Unknown, read.Frames[0].TimeSource); + Assert.Equal((FrameFlags)4096, read.Frames[0].Flags); + + ManifestStore.WriteSequence(path, read with { Title = "Comet 2026" }); + var json = File.ReadAllText(path); + foreach (var kept in new[] { "futureBlock", "sensorTemp", "aiScore", "renders", "4096", "1024" }) Assert.Contains(kept, json); + Assert.Equal("Comet 2026", ManifestStore.ReadSequence(path).Title); + } + + [Test] + public void ANewerSchemaIsReadableButNeverRewritten() + { + var path = SidecarPath(); + ManifestStore.WriteSequence(path, Sample()); + var newer = ManifestStore.ReadSequence(path) with { SchemaVersion = SequenceManifest.CurrentSchemaVersion + 1 }; + var before = File.ReadAllBytes(path); + Assert.Throws(() => ManifestStore.WriteSequence(path, newer)); + Assert.BytesEqual(before, File.ReadAllBytes(path)); + } + + [Test] + public void DamagedJsonIsReportedAsInvalidData() + { + var path = SidecarPath(); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, "{ \"id\": "); + Assert.Throws(() => ManifestStore.ReadSequence(path)); + File.WriteAllText(path, "{ \"title\": \"no id\" }"); + Assert.Throws(() => ManifestStore.ReadSequence(path)); + } + + [Test] + public void ACrashDuringAWriteLeavesThePreviousSidecarIntact() + { + var path = SidecarPath(); + var first = Sample(); + ManifestStore.WriteSequence(path, first); + // Simulated crash: the next version was half written to the temporary sibling and never renamed. + var torn = AtomicFile.TempPathFor(path); + File.WriteAllBytes(torn, Encoding.UTF8.GetBytes(Canonical(first with { Title = "Lost" })[..40])); + + Assert.Equal(Canonical(first), Canonical(ManifestStore.ReadSequence(path))); + ManifestStore.WriteSequence(path, first with { Title = "Second" }); + Assert.Equal("Second", ManifestStore.ReadSequence(path).Title); + Assert.FileExists(torn); + Assert.Equal(2, Directory.GetFiles(Path.GetDirectoryName(path)!).Length); + } + + [Test] + public void CreatingASidecarClaimsTheFolderOnce() + { + var path = SidecarPath(); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var a = Sample(); + var b = Sample(); + Assert.True(ManifestStore.TryCreateSequence(path, a)); + Assert.False(ManifestStore.TryCreateSequence(path, b)); + Assert.Equal(a.Id, ManifestStore.ReadSequence(path).Id); + Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(path)!)); + } + + [Test] + public void ArchiveManifestRoundTripsAndKeepsUnknownFields() + { + var path = Path.Combine(context.CreateTempDirectory(), "archive.json"); + File.WriteAllText(path, $$""" + { "schemaVersion": 1, "id": "{{Guid.CreateVersion7()}}", "name": "Main", "created": "2026-10-02T09:00:00+00:00", + "layoutTemplates": { "folder": "{year}/{title}", "file": "{n:00000}" }, "proxyPolicy": { "longEdge": 2560, "quality": 90 }, + "syncPolicy": "mirror" } + """); + var read = ManifestStore.ReadArchive(path); + Assert.Equal("{year}/{title}", read.LayoutTemplates.Folder); + Assert.Equal(2560, read.ProxyPolicy.LongEdge); + ManifestStore.WriteArchive(path, read with { Name = "Studio" }); + Assert.Contains("syncPolicy", File.ReadAllText(path)); + Assert.Equal("Studio", ManifestStore.ReadArchive(path).Name); + } +} diff --git a/tests/Titano.Core.Tests/Archiving/TestPhotos.cs b/tests/Titano.Core.Tests/Archiving/TestPhotos.cs new file mode 100644 index 00000000..c0cb2db2 --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/TestPhotos.cs @@ -0,0 +1,51 @@ +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; + +namespace Titano.Core.Tests.Archiving; + +/// Small real JPEGs with Exif (capture time, camera) made by the in-house encoder; each seed gives distinct bytes. +internal static class TestPhotos +{ + public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431", + int width = 48, int height = 32, int orientation = 1) + { + using var image = new ImageBuffer(width, height); + var random = new Random(seed); + for (var c = 0; c < 3; c++) + { + var plane = image.Plane(c); + var level = (float)random.NextDouble(); + for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % width) / width); + } + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata + { + Make = make, + Model = model, + BodySerial = serial, + CapturedAt = captured, + ExposureTime = 1.0 / 125, + FNumber = 8, + Iso = 100, + Orientation = orientation, + }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + public static string Write(string path, DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431") + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, Jpeg(captured, seed, make, model, serial)); + return path; + } + + /// Writes photos named IMG_0001.JPG… taken every . + public static List WriteSeries(string directory, DateTime start, int count, TimeSpan interval, int firstSeed = 1, string prefix = "IMG_", + string make = "Canon", string model = "Canon EOS R5", int firstNumber = 1) + { + var paths = new List(count); + for (var i = 0; i < count; i++) + paths.Add(Write(Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG"), start + interval * i, firstSeed + i, make, model)); + return paths; + } +} diff --git a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs new file mode 100644 index 00000000..b360aafd --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs @@ -0,0 +1,165 @@ +using Titano.Core.Archiving; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class ThumbPackTests(TestContext context) +{ + private static byte[] Payload(int seed, int length = 300) + { + var data = new byte[length]; + new Random(seed).NextBytes(data); + return data; + } + + private static byte[] Hash(int seed) => ThumbPack.Fingerprint(null, $"originals/{seed}.jpg", seed); + + private string NewPack() => Path.Combine(context.CreateTempDirectory(), ".titano", "thumbs.pack"); + + private static void Fill(string path, int count, int seedOffset = 0) + { + using var writer = ThumbPackWriter.Open(path); + for (var i = 0; i < count; i++) writer.Append(i, 320, 213, Hash(i + seedOffset), Payload(i + seedOffset)); + } + + [Test] + public void EntriesAreReadBackByFrameIndex() + { + var path = NewPack(); + Fill(path, 100); + using var reader = ThumbPackReader.Open(path); + Assert.Equal(100, reader.Entries.Count); + Assert.BytesEqual(Payload(57), reader.Read(57)); + Assert.BytesEqual(Hash(57), reader.Entries[57].SourceHash); + Assert.Equal(320, reader.Entries[57].Width); + Assert.Null(reader.Read(100)); + } + + [Test] + public void AppendingAgainReplacesAFrameAndKeepsTheOthers() + { + var path = NewPack(); + Fill(path, 3); + using (var writer = ThumbPackWriter.Open(path)) + { + Assert.Equal(3, writer.Entries.Count); + writer.Append(1, 200, 100, Hash(99), Payload(99)); + writer.Append(3, 200, 100, Hash(3), Payload(3)); + } + using var reader = ThumbPackReader.Open(path); + Assert.Equal(4, reader.Entries.Count); + Assert.BytesEqual(Payload(99), reader.Read(1)); + Assert.BytesEqual(Payload(0), reader.Read(0)); + Assert.BytesEqual(Payload(3), reader.Read(3)); + } + + [Test] + public void AnIncompleteTailIsIgnoredAndTruncatedByTheNextWriter() + { + var path = NewPack(); + Fill(path, 2); + var committed = new FileInfo(path).Length; + // Simulated crash mid-append: an entry header and part of its payload, no index. + using (var stream = new FileStream(path, FileMode.Append)) + { + stream.Write("TENT"u8); + stream.Write(BitConverter.GetBytes(2)); + stream.Write(new byte[30]); + } + using (var reader = ThumbPackReader.Open(path)) + { + Assert.Equal(2, reader.Entries.Count); + Assert.BytesEqual(Payload(1), reader.Read(1)); + } + using (var writer = ThumbPackWriter.Open(path)) + { + Assert.Equal(committed, new FileInfo(path).Length); + writer.Append(2, 320, 213, Hash(2), Payload(2)); + } + using var after = ThumbPackReader.Open(path); + Assert.Equal(3, after.Entries.Count); + Assert.BytesEqual(Payload(2), after.Read(2)); + } + + [Test] + public void EntriesWrittenAfterTheLastIndexAreFoundWithoutIt() + { + var path = NewPack(); + Fill(path, 5); + // Simulated crash after new entries overwrote the old index but before the new index was written. + var withoutIndex = File.ReadAllBytes(path); + using (var reader = ThumbPackReader.Open(path)) + { + var end = reader.Entries.Values.Max(e => e.Offset + e.Length); + File.WriteAllBytes(path, withoutIndex[..(int)end]); + } + using var scanned = ThumbPackReader.Open(path); + Assert.Equal(5, scanned.Entries.Count); + for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), scanned.Read(i)); + } + + [Test] + public void ADamagedThumbnailReadsAsMissing() + { + var path = NewPack(); + Fill(path, 3); + long offset; + using (var reader = ThumbPackReader.Open(path)) offset = reader.Entries[1].Offset; + using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) + { + stream.Position = offset + 10; + var b = stream.ReadByte(); + stream.Position = offset + 10; + stream.WriteByte((byte)(b ^ 0xFF)); + } + using var damaged = ThumbPackReader.Open(path); + Assert.Null(damaged.Read(1)); + Assert.BytesEqual(Payload(2), damaged.Read(2)); + } + + [Test] + public void AnOpenReaderKeepsReadingWhileAWriterAppends() + { + var path = NewPack(); + Fill(path, 3); + using var reader = ThumbPackReader.Open(path); + using (var writer = ThumbPackWriter.Open(path)) + { + for (var i = 3; i < 50; i++) writer.Append(i, 320, 213, Hash(i), Payload(i, 2000)); + } + Assert.BytesEqual(Payload(0), reader.Read(0)); + Assert.BytesEqual(Payload(2), reader.Read(2)); + using var fresh = ThumbPackReader.Open(path); + Assert.Equal(50, fresh.Entries.Count); + } + + [Test] + public void CompactionDropsReplacedEntries() + { + var path = NewPack(); + for (var round = 0; round < 4; round++) Fill(path, 20, seedOffset: round * 100); + long garbage; + using (var reader = ThumbPackReader.Open(path)) garbage = reader.GarbageBytes; + Assert.Greater(garbage, 0); + var before = new FileInfo(path).Length; + ThumbPackWriter.Compact(path); + Assert.Equal(before - garbage, new FileInfo(path).Length); + using var compacted = ThumbPackReader.Open(path); + Assert.Equal(0L, compacted.GarbageBytes); + for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i)); + } + + [Test] + public void AMissingOrForeignFileReadsAsEmpty() + { + var path = NewPack(); + using (var none = ThumbPackReader.Open(path)) Assert.Equal(0, none.Entries.Count); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, "not a pack"); + using (var foreign = ThumbPackReader.Open(path)) Assert.Equal(0, foreign.Entries.Count); + Fill(path, 1); + using var rewritten = ThumbPackReader.Open(path); + Assert.Equal(1, rewritten.Entries.Count); + } +} diff --git a/tests/Titano.Core.Tests/Import/ImportExecutorTests.cs b/tests/Titano.Core.Tests/Import/ImportExecutorTests.cs new file mode 100644 index 00000000..8fdb8f4d --- /dev/null +++ b/tests/Titano.Core.Tests/Import/ImportExecutorTests.cs @@ -0,0 +1,179 @@ +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Tests.Archiving; +using Titano.Testing; +using static Titano.Core.Tests.Import.ImportTestKit; + +namespace Titano.Core.Tests.Import; + +[Category("import")] +public sealed class ImportExecutorTests(TestContext context) +{ + private static readonly TimeSpan Interval = TimeSpan.FromSeconds(10); + + private string Card(int frames = 70) + { + var card = context.CreateTempDirectory("card"); + TestPhotos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames, Interval); + return card; + } + + private static Task RunAsync(ImportPlan plan, IImportSink sink, int startAt = 0, Func? committed = null, + CancellationToken cancellationToken = default) => + new ImportExecutor(plan, sink).RunAsync(startAt, verify: true, committed ?? (_ => ValueTask.CompletedTask), null, cancellationToken); + + [Test] + public async Task VerifiedCopyHashesPreservesTheTimeAndNeverOverwrites() + { + var dir = context.CreateTempDirectory(); + var source = TestPhotos.Write(Path.Combine(dir, "a.jpg"), Start, 1); + File.SetLastWriteTimeUtc(source, new DateTime(2026, 9, 28, 16, 0, 0, DateTimeKind.Utc)); + var target = Path.Combine(dir, "copy", "a.jpg"); + var sha = await VerifiedCopy.CopyAsync(source, target, verify: true); + Assert.Equal(await ContentHash.Sha256Async(source), sha); + Assert.BytesEqual(File.ReadAllBytes(source), File.ReadAllBytes(target)); + Assert.Equal(File.GetLastWriteTimeUtc(source), File.GetLastWriteTimeUtc(target)); + + var other = TestPhotos.Write(Path.Combine(dir, "b.jpg"), Start, 2); + await Assert.ThrowsAsync(() => VerifiedCopy.CopyAsync(other, target, verify: true)); + Assert.BytesEqual(File.ReadAllBytes(source), File.ReadAllBytes(target)); + await Assert.ThrowsAsync(() => VerifiedCopy.CopyAsync(Path.Combine(dir, "gone.jpg"), Path.Combine(dir, "c.jpg"), true)); + Assert.FileMissing(Path.Combine(dir, "c.jpg")); + } + + [Test] + public async Task APlanIsCopiedExactlyIntoAFolder() + { + var card = Card(); + var destination = context.CreateTempDirectory("dest"); + var plan = await PlanAsync(card, destination); + var batches = new List(); + var final = await RunAsync(plan, new FolderImportSink(destination), committed: b => + { + batches.Add(b); + return ValueTask.CompletedTask; + }); + Assert.Equal(70, final.Copied); + Assert.Equal(plan.CopyBytes, final.BytesDone); + Assert.SequenceEqual(plan.Items.Select(i => plan.Sessions[0].Folder + "/" + i.FileName).Order(StringComparer.Ordinal), Files(destination)); + Assert.Equal(70, batches[^1].NextItem); + Assert.True(batches.Count >= 3); + Assert.Empty(Directory.EnumerateFiles(new FolderImportSink(destination).StagingDirectory)); + foreach (var item in plan.Items) + { + Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), + File.ReadAllBytes(Path.Combine(destination, plan.Sessions[0].Folder, item.FileName))); + } + } + + [Test] + public async Task AnInterruptedImportResumesFromItsLastBatch() + { + var card = Card(); + var destination = context.CreateTempDirectory("dest"); + var plan = await PlanAsync(card, destination); + using var cts = new CancellationTokenSource(); + var checkpoint = 0; + await Assert.ThrowsAsync(() => RunAsync(plan, new FolderImportSink(destination), committed: b => + { + checkpoint = b.NextItem; + cts.Cancel(); + return ValueTask.CompletedTask; + }, cancellationToken: cts.Token)); + Assert.True(checkpoint is > 0 and < 70); + var final = await RunAsync(plan, new FolderImportSink(destination), startAt: checkpoint); + Assert.Equal(70 - checkpoint, final.Copied); + + var reference = context.CreateTempDirectory("reference"); + await RunAsync(await PlanAsync(card, reference), new FolderImportSink(reference)); + Assert.SequenceEqual(Files(reference), Files(destination)); + } + + [Test] + public async Task RepeatingCommittedItemsYieldsDuplicatesNotSecondCopies() + { + var card = Card(20); + var destination = context.CreateTempDirectory("dest"); + var plan = await PlanAsync(card, destination); + await RunAsync(plan, new FolderImportSink(destination)); + var again = await RunAsync(plan, new FolderImportSink(destination), startAt: 10); + Assert.Equal(0, again.Copied); + Assert.Equal(10, again.Skipped); + Assert.Equal(20, Files(destination).Count); + } + + [Test] + public async Task ARemovedCardStopsTheImportWhichResumesOnReinsertion() + { + var card = Card(); + var destination = context.CreateTempDirectory("dest"); + var plan = await PlanAsync(card, destination); + var aside = card + "-removed"; + var checkpoint = 0; + await Assert.ThrowsAsync(() => RunAsync(plan, new FolderImportSink(destination), committed: b => + { + checkpoint = b.NextItem; + if (Directory.Exists(card)) Directory.Move(card, aside); // the card is pulled after the first batch + return ValueTask.CompletedTask; + })); + Assert.Equal(checkpoint, Files(destination).Count); + Assert.Empty(Directory.EnumerateFiles(new FolderImportSink(destination).StagingDirectory)); + + Directory.Move(aside, card); + var final = await RunAsync(plan, new FolderImportSink(destination), startAt: checkpoint); + Assert.Equal(70 - checkpoint, final.Copied); + Assert.Equal(70, Files(destination).Count); + } + + [Test] + public async Task AFileThatChangedSinceThePlanFailsAloneAndTheRestIsCopied() + { + var card = Card(20); + var destination = context.CreateTempDirectory("dest"); + var plan = await PlanAsync(card, destination); + File.AppendAllText(Path.Combine(card, plan.Sessions[0].Items[5].SourcePath), "edited"); + var failed = new List(); + var final = await RunAsync(plan, new FolderImportSink(destination), committed: b => + { + failed.AddRange(b.Results.Where(r => r.Status == ImportItemStatus.Failed)); + return ValueTask.CompletedTask; + }); + Assert.Equal(19, final.Copied); + Assert.Equal(1, final.Failed); + Assert.Equal(plan.Sessions[0].Items[5].SourcePath, failed.Single().Item.SourcePath); + } + + [Test] + public async Task ErasingDeletesOnlyFilesWhoseCardAndCopyStillMatch() + { + var card = Card(6); + var destination = context.CreateTempDirectory("dest"); + var plan = await PlanAsync(card, destination, new ImportOptions { MinimumSessionFrames = 1 }); + var results = new List(); + await RunAsync(plan, new FolderImportSink(destination), committed: b => + { + results.AddRange(b.Results); + return ValueTask.CompletedTask; + }); + var candidates = results.Select(r => new EraseCandidate(r.Item.SourcePath, r.Item.Size, r.Sha256!, Path.Combine(destination, r.Commit!.RelativePath))) + .ToList(); + // Copy of item 1 damaged after import, card file of item 2 edited, copy of item 3 deleted. + var damaged = File.ReadAllBytes(candidates[1].CopyPath); + damaged[^10] ^= 0xFF; + File.WriteAllBytes(candidates[1].CopyPath, damaged); + File.AppendAllText(Path.Combine(card, candidates[2].SourcePath), "x"); + File.Delete(candidates[3].CopyPath); + + var erase = await CardEraser.PlanAsync(card, candidates); + Assert.SequenceEqual( + [EraseVerdict.Delete, EraseVerdict.CopyMismatch, EraseVerdict.SourceChanged, EraseVerdict.CopyMissing, EraseVerdict.Delete, EraseVerdict.Delete], + erase.Items.Select(i => i.Verdict)); + Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count()); + + var done = CardEraser.Execute(erase); + Assert.Equal(3, done.Deleted); + Assert.Equal(3, done.Kept.Count); + var left = Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/')).Order(); + Assert.SequenceEqual(candidates.Skip(1).Take(3).Select(c => c.SourcePath).Order(), left); + } +} diff --git a/tests/Titano.Core.Tests/Import/ImportPlannerTests.cs b/tests/Titano.Core.Tests/Import/ImportPlannerTests.cs new file mode 100644 index 00000000..ad45c26e --- /dev/null +++ b/tests/Titano.Core.Tests/Import/ImportPlannerTests.cs @@ -0,0 +1,160 @@ +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Tests.Archiving; +using Titano.Testing; +using static Titano.Core.Tests.Import.ImportTestKit; + +namespace Titano.Core.Tests.Import; + +[Category("import")] +public sealed class ImportPlannerTests(TestContext context) +{ + private static readonly TimeSpan Interval = TimeSpan.FromSeconds(10); + + [Test] + public async Task ADcimCardIsPlannedWithTheDefaultTemplates() + { + var card = context.CreateTempDirectory("card"); + TestPhotos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 10, Interval); + File.WriteAllBytes(Path.Combine(card, "DCIM", "100CANON", "._IMG_0001.JPG"), [0, 5, 22]); + File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "x"); + TestPhotos.Write(Path.Combine(card, "MISC", "IMG_9999.JPG"), Start, 99); + + var plan = await PlanAsync(card, context.CreateTempDirectory("archive")); + var session = plan.Sessions.Single(); + Assert.Equal("2026/2026-09-28 Canon EOS R5", session.Folder); + Assert.Equal(10, plan.FileCount); + Assert.Equal(10, plan.CopyCount); + Assert.SequenceEqual( + ["2026-09-28_18-00-00_0001.JPG", "2026-09-28_18-00-10_0002.JPG", "2026-09-28_18-00-20_0003.JPG"], + session.Items.Take(3).Select(i => i.FileName)); + Assert.Equal("DCIM/100CANON/IMG_0001.JPG", session.Items[0].SourcePath); + Assert.Equal(plan.Items.Sum(i => i.Size), plan.CopyBytes); + Assert.NotNull(plan.AvailableBytes); + Assert.True(plan.HasEnoughSpace); + Assert.Empty(plan.Problems); + } + + [Test] + public async Task AFlatFolderIncludesSubfoldersOnlyWhenAsked() + { + var card = context.CreateTempDirectory("card"); + TestPhotos.WriteSeries(card, Start, 8, Interval); + TestPhotos.WriteSeries(Path.Combine(card, "more"), Start.AddSeconds(80), 8, Interval, firstSeed: 50, firstNumber: 9); + var destination = context.CreateTempDirectory("archive"); + Assert.Equal(16, (await PlanAsync(card, destination)).FileCount); + Assert.Equal(8, (await PlanAsync(card, destination, new ImportOptions { IncludeSubfolders = false })).FileCount); + } + + [Test] + public async Task SessionsAreSplitPerCameraBodyAndNumberedFromOne() + { + var card = context.CreateTempDirectory("card"); + var dcim = Path.Combine(card, "DCIM", "100MEDIA"); + TestPhotos.WriteSeries(dcim, Start, 10, Interval, prefix: "CAN_"); + TestPhotos.WriteSeries(dcim, Start.AddHours(2), 10, Interval, firstSeed: 20, prefix: "CAN_", firstNumber: 11); + TestPhotos.WriteSeries(dcim, Start.AddSeconds(5), 9, Interval, firstSeed: 40, prefix: "NIK_", make: "NIKON CORPORATION", model: "NIKON Z 8"); + + var plan = await PlanAsync(card, context.CreateTempDirectory("archive")); + Assert.SequenceEqual( + ["2026/2026-09-28 Canon EOS R5", "2026/2026-09-28 Nikon Z 8", "2026/2026-09-28 Canon EOS R5 (2)"], + plan.Sessions.Select(s => s.Folder)); + Assert.SequenceEqual([1, 2, 3], plan.Sessions.Select(s => s.Number)); + Assert.Equal("2026-09-28_20-00-00_0001.JPG", plan.Sessions[2].Items[0].FileName); + Assert.Equal(3, plan.Sessions.Select(s => s.SessionKey).Distinct().Count()); + Assert.True(plan.Sessions.All(s => s.Items.All(i => i.SourcePath.Contains(s.CameraModel == "NIKON Z 8" ? "NIK_" : "CAN_")))); + } + + [Test] + public async Task NamesTakenTwiceGetASuffix() + { + var card = context.CreateTempDirectory("card"); + TestPhotos.WriteSeries(card, Start, 3, Interval); + var plan = await PlanAsync(card, context.CreateTempDirectory("archive"), new ImportOptions { FileTemplate = "{date}", MinimumSessionFrames = 1 }); + Assert.SequenceEqual(["2026-09-28.JPG", "2026-09-28_2.JPG", "2026-09-28_3.JPG"], plan.Sessions[0].Items.Select(i => i.FileName)); + } + + [Test] + public async Task ImportedAndArchivedPhotosAreSkippedAfterAHashMatch() + { + var card = context.CreateTempDirectory("card"); + var files = TestPhotos.WriteSeries(card, Start, 10, Interval); + var lookup = new FakeLookup(); + lookup.Imported.Add("IMG_0001.JPG"); + var archived = await ContentHash.Sha256Async(files[1]); + lookup.Archive.Add((new FileInfo(files[1]).Length, Start + Interval, new ArchiveMatch(Guid.NewGuid(), archived))); + lookup.Archive.Add((new FileInfo(files[2]).Length, Start + 2 * Interval, new ArchiveMatch(Guid.NewGuid(), new string('0', 64)))); + + var plan = await PlanAsync(card, context.CreateTempDirectory("archive"), lookup: lookup); + var items = plan.Sessions[0].Items; + Assert.Equal(ImportAction.SkipImported, items[0].Action); + Assert.Equal(ImportAction.SkipInArchive, items[1].Action); + Assert.Equal(lookup.Archive[0].Match.FrameId, items[1].ExistingFrameId); + Assert.Equal(ImportAction.Copy, items[2].Action); + Assert.Equal(await ContentHash.Sha256Async(files[2]), items[2].Sha256); + Assert.Null(items[3].Sha256); + Assert.Equal(8, plan.CopyCount); + Assert.Equal("2026-09-28_18-00-10_0002.JPG", items[1].FileName); + + var again = await PlanAsync(card, context.CreateTempDirectory("archive"), new ImportOptions { Duplicates = DuplicatePolicy.ImportAgain }, lookup); + Assert.Equal(ImportAction.Copy, again.Sessions[0].Items[1].Action); + Assert.Equal(lookup.Archive[0].Match.FrameId, again.Sessions[0].Items[1].ExistingFrameId); + } + + [Test] + public async Task ReplanningGivesStableKeysThatTitlesAndEarlierFoldersAttachTo() + { + var card = context.CreateTempDirectory("card"); + TestPhotos.WriteSeries(card, Start, 10, Interval); + var destination = context.CreateTempDirectory("archive"); + var first = await PlanAsync(card, destination); + var key = first.Sessions[0].SessionKey; + Assert.Equal(key, (await PlanAsync(card, destination)).Sessions[0].SessionKey); + + var titled = await PlanAsync(card, destination, new ImportOptions + { + FolderTemplate = "{year}/{date} {title}", + Titles = new Dictionary { [key] = "Harbour sunset" }, + }); + Assert.Equal("2026/2026-09-28 Harbour sunset", titled.Sessions[0].Folder); + Assert.Equal("Harbour sunset", titled.Sessions[0].Title); + + var lookup = new FakeLookup(); + lookup.SessionFolders[key] = "2026/earlier import"; + Assert.Equal("2026/earlier import", (await PlanAsync(card, destination, lookup: lookup)).Sessions[0].Folder); + + var taken = new FakeLookup(); + taken.TakenFolders.Add("2026/2026-09-28 Canon EOS R5"); + Assert.Equal("2026/2026-09-28 Canon EOS R5 (2)", (await PlanAsync(card, destination, lookup: taken)).Sessions[0].Folder); + } + + [Test] + public async Task UserMergeAndSplitReshapeSessions() + { + var card = context.CreateTempDirectory("card"); + TestPhotos.WriteSeries(card, Start, 10, Interval); + TestPhotos.WriteSeries(card, Start.AddHours(3), 10, Interval, firstSeed: 30, firstNumber: 11); + var destination = context.CreateTempDirectory("archive"); + Assert.Equal(2, (await PlanAsync(card, destination)).Sessions.Count); + var merged = await PlanAsync(card, destination, new ImportOptions { MergeAt = new HashSet { "IMG_0011.JPG" } }); + Assert.Equal(20, merged.Sessions.Single().Items.Count); + Assert.Equal("2026-09-28_21-00-00_0011.JPG", merged.Sessions[0].Items[10].FileName); + var split = await PlanAsync(card, destination, new ImportOptions { SplitBefore = new HashSet { "IMG_0006.JPG" } }); + Assert.SequenceEqual([5, 5, 10], split.Sessions.Select(s => s.Items.Count)); + } + + [Test] + public async Task APhotoWithoutExifIsPlannedAtItsFileTime() + { + var card = context.CreateTempDirectory("card"); + TestPhotos.WriteSeries(card, Start, 8, Interval); + var bare = Path.Combine(card, "BARE.JPG"); + File.WriteAllBytes(bare, [0xFF, 0xD8, 0xFF, 0xD9]); + File.SetLastWriteTime(bare, Start.AddSeconds(85)); + var plan = await PlanAsync(card, context.CreateTempDirectory("archive")); + var item = plan.Items.Single(i => i.SourcePath == "BARE.JPG"); + Assert.Equal(Start.AddSeconds(85), item.CapturedAt); + // No camera identity: it forms its own body group rather than joining the Canon session. + Assert.SequenceEqual([8, 1], plan.Sessions.Select(s => s.Items.Count)); + } +} diff --git a/tests/Titano.Core.Tests/Import/ImportTestKit.cs b/tests/Titano.Core.Tests/Import/ImportTestKit.cs new file mode 100644 index 00000000..887c57fc --- /dev/null +++ b/tests/Titano.Core.Tests/Import/ImportTestKit.cs @@ -0,0 +1,48 @@ +using Titano.Core.Import; + +namespace Titano.Core.Tests.Import; + +/// An answered from in-memory sets. +internal sealed class FakeLookup : IImportLookup +{ + public HashSet Imported { get; } = []; + public List<(long Size, DateTime Captured, ArchiveMatch Match)> Archive { get; } = []; + public Dictionary SessionFolders { get; } = []; + public HashSet TakenFolders { get; } = new(StringComparer.OrdinalIgnoreCase); + public int CandidateQueries { get; private set; } + + public Task WasImportedAsync(CardFile file, CancellationToken cancellationToken) => Task.FromResult(Imported.Contains(file.RelativePath)); + + public Task> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) + { + CandidateQueries++; + return Task.FromResult>(Archive.Where(a => a.Size == size && a.Captured == capturedAt).Select(a => a.Match).ToList()); + } + + public Task FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) => + Task.FromResult(SessionFolders.GetValueOrDefault(sessionKey)); + + public bool IsFolderTaken(string folder) => TakenFolders.Contains(folder); +} + +internal static class ImportTestKit +{ + public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); + + /// Plans an import of into . + public static async Task PlanAsync(string card, string destination, ImportOptions? options = null, FakeLookup? lookup = null) + { + options ??= new ImportOptions(); + var problems = new List(); + var photos = CardScanner.ReadMetadata(card, CardScanner.Find(card, options.IncludeSubfolders), problems); + return await ImportPlanner.PlanAsync(card, destination, photos, problems, options, lookup ?? new FakeLookup()); + } + + /// Every file below (relative, forward slashes), excluding Titano's metadata folders. + public static List Files(string root) => + Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories) + .Select(f => Path.GetRelativePath(root, f).Replace('\\', '/')) + .Where(f => !f.StartsWith(".titano/", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal) + .ToList(); +} diff --git a/tests/Titano.Core.Tests/Import/SessionSplitterTests.cs b/tests/Titano.Core.Tests/Import/SessionSplitterTests.cs new file mode 100644 index 00000000..c7d84e54 --- /dev/null +++ b/tests/Titano.Core.Tests/Import/SessionSplitterTests.cs @@ -0,0 +1,73 @@ +using Titano.Core.Import; +using Titano.Testing; + +namespace Titano.Core.Tests.Import; + +[Category("import")] +public sealed class SessionSplitterTests +{ + private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); + private static readonly TimeSpan Gap = TimeSpan.FromMinutes(45); + + /// Capture times from a compact description: runs of frames 10 s apart, separated by pauses in minutes. + private static List Times(params (int Frames, int PauseBeforeMinutes)[] runs) + { + var times = new List(); + var t = Start; + foreach (var (frames, pause) in runs) + { + t = t.AddMinutes(pause); + for (var i = 0; i < frames; i++) + { + times.Add(t); + t = t.AddSeconds(10); + } + } + return times; + } + + private static string Describe(IReadOnlyList ranges) => string.Join(",", ranges.Select(r => r.Count)); + + [Theory] + [Case(new[] { 30 }, new[] { 0 }, "30", Name = "one session")] + [Case(new[] { 30, 40 }, new[] { 0, 60 }, "30,40", Name = "pause longer than the gap splits")] + [Case(new[] { 30, 40 }, new[] { 0, 44 }, "70", Name = "pause shorter than the gap does not")] + [Case(new[] { 3, 30 }, new[] { 0, 90 }, "33", Name = "leading test shots join the next session")] + [Case(new[] { 30, 3 }, new[] { 0, 90 }, "33", Name = "trailing test shots join the previous session")] + [Case(new[] { 30, 2, 30 }, new[] { 0, 90, 90 }, "32,30", Name = "small middle group joins the previous session")] + [Case(new[] { 2, 3 }, new[] { 0, 90 }, "5", Name = "only small groups make one session")] + public void AutomaticSessions(int[] frames, int[] pauses, string expected) + { + var times = Times(frames.Zip(pauses).ToArray()); + Assert.Equal(expected, Describe(SessionSplitter.Split(times, Gap, 8))); + } + + [Test] + public void APauseOfExactlyTheGapDoesNotSplit() + { + var times = new List { Start }; + for (var i = 1; i < 10; i++) times.Add(Start.AddSeconds(i)); + for (var i = 0; i < 10; i++) times.Add(times[9] + Gap + TimeSpan.FromSeconds(i)); + Assert.Equal("20", Describe(SessionSplitter.Split(times, Gap, 8))); + } + + [Test] + public void UserSplitsAndMergesOverrideTheRules() + { + var times = Times((30, 0), (40, 60)); + // Split before frame 25 (a 5-frame group the user wants on its own) and merge the automatic break at 30. + var ranges = SessionSplitter.Split(times, Gap, 8, i => i == 25 ? true : i == 30 ? false : null); + Assert.Equal("25,45", Describe(ranges)); + Assert.Equal(25, ranges[1].Start); + } + + [Test] + public void AForcedSmallSessionIsNotMergedAway() + { + var times = Times((30, 0)); + Assert.Equal("28,2", Describe(SessionSplitter.Split(times, Gap, 8, i => i == 28 ? true : null))); + } + + [Test] + public void NoFramesNoSessions() => Assert.Empty(SessionSplitter.Split([], Gap, 8)); +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs new file mode 100644 index 00000000..299674ff --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveFixture.cs @@ -0,0 +1,162 @@ +using System.Security.Cryptography; +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Jobs; +using Titano.Testing; + +namespace Titano.Data.Tests.Archiving; + +/// A temporary archive root with its SQLite index and the archive service. +internal sealed class ArchiveFixture : IAsyncDisposable +{ + public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); + + private readonly TestContext _context; + private readonly List _owned = []; + + private ArchiveFixture(TestContext context, TitanoDatabase db, ArchiveService service, Archive archive) + { + _context = context; + Db = db; + Service = service; + Archive = archive; + _owned.Add(db); + } + + public TitanoDatabase Db { get; private set; } + public ArchiveService Service { get; private set; } + public Archive Archive { get; private set; } + public ArchivePaths Paths => new(Archive.RootPath); + public CancellationToken Ct => _context.Cancellation; + + public static async Task CreateAsync(TestContext context, string? root = null) + { + var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(context.CreateTempDirectory("db"), "titano.db")), context.Cancellation); + var service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); + var archive = await service.AdoptAsync(root ?? context.CreateTempDirectory("archive"), "Main", context.Cancellation); + return new ArchiveFixture(context, db, service, archive); + } + + /// Switches to a brand-new empty index of the same archive (the "lost database" case). + public async Task ReopenWithEmptyIndexAsync() + { + var db = await TitanoDatabase.OpenAsync(DatabaseOptions.Sqlite(Path.Combine(_context.CreateTempDirectory("db2"), "titano.db")), Ct); + _owned.Add(db); + Db = db; + Service = new ArchiveService(db, new ArchiveServiceOptions { Owner = "test", LeaseWait = TimeSpan.FromSeconds(20) }); + Archive = await Service.OpenAsync(Archive.RootPath, Ct); + } + + public Task ScanAsync(ScanMode mode = ScanMode.Incremental) => + Service.Indexer.ScanAsync(Archive, mode, null, _ => ValueTask.CompletedTask, Ct); + + /// A job engine with the archive handlers, started; disposed with the fixture. + public async Task StartEngineAsync() + { + var engine = new JobEngine(Db, Service.CreateJobHandlers(), + new JobEngineOptions { Workers = 1, WorkerId = "test", PollInterval = TimeSpan.FromMilliseconds(50), CheckpointInterval = TimeSpan.FromMilliseconds(50) }); + _owned.Insert(0, engine); + await engine.StartAsync(Ct); + return engine; + } + + public async Task WaitForAsync(Guid jobId, JobState state, int seconds = 30) + { + var until = DateTime.UtcNow.AddSeconds(seconds); + Job? job = null; + while (DateTime.UtcNow < until) + { + job = await Db.Jobs.GetAsync(jobId, Ct); + if (job?.State == state) return job; + if (job?.State is JobState.Failed && state != JobState.Failed) Assert.Fail($"Job failed: {job.Error}\n{job.LogTail}"); + await Task.Delay(20, Ct); + } + Assert.Fail($"Job {jobId} did not reach {state} (now {job?.State}): {job?.Error}\n{job?.LogTail}"); + return null!; + } + + /// Queues a job directly with a checkpoint, as a crashed earlier attempt would have left it. + public async Task EnqueueWithCheckpointAsync(JobEngine engine, Titano.Data.Archiving.Jobs.ArchiveJobRequest request, string checkpoint) + { + var job = new Job + { + Id = Guid.CreateVersion7(), + Kind = request.Kind, + TargetIds = request.Targets, + ParametersJson = request.ParametersJson, + State = JobState.Queued, + Checkpoint = checkpoint, + Created = DateTimeOffset.UtcNow, + }; + await Db.Jobs.EnqueueAsync(job, Ct); + engine.Wake(); + return job; + } + + /// Stages a verified upload-like file for ingest: returns its path, size and SHA-256. + public (string Path, long Size, string Sha256) Stage(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5") + { + var staging = Path.Combine(Paths.StagingDirectory, "uploads"); + Directory.CreateDirectory(staging); + var bytes = Photos.Jpeg(captured, seed, make, model); + var path = Path.Combine(staging, $"{Guid.NewGuid():N}.part"); + File.WriteAllBytes(path, bytes); + return (path, bytes.Length, Convert.ToHexStringLower(SHA256.HashData(bytes))); + } + + public async ValueTask DisposeAsync() + { + foreach (var owned in _owned) await owned.DisposeAsync(); + } +} + +/// Small real JPEGs with Exif made by the in-house encoder; each seed gives distinct bytes. +internal static class Photos +{ + public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5") + { + using var image = new ImageBuffer(48, 32); + var random = new Random(seed); + for (var c = 0; c < 3; c++) + { + var plane = image.Plane(c); + var level = (float)random.NextDouble(); + for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % 48) / 48f); + } + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata { Make = make, Model = model, BodySerial = "0431", CapturedAt = captured, Iso = 100 }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + public static List WriteSeries(string directory, DateTime start, int count, int firstSeed = 1, string prefix = "IMG_", int firstNumber = 1, + string make = "Canon", string model = "Canon EOS R5") + { + Directory.CreateDirectory(directory); + var paths = new List(count); + for (var i = 0; i < count; i++) + { + var path = Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG"); + File.WriteAllBytes(path, Jpeg(start.AddSeconds(10 * i), firstSeed + i, make, model)); + paths.Add(path); + } + return paths; + } + + /// The index as comparable text: sequences and frames ordered by folder and index. + public static async Task DumpIndexAsync(TitanoDatabase db, Guid archiveId) + { + var lines = new List(); + foreach (var location in await db.Sequences.ListLocationsAsync(archiveId)) + { + var s = (await db.Sequences.GetAsync(location.Id))!; + lines.Add($"S {s.Id} {s.Folder} '{s.Title}' {s.CameraModel} {s.FirstCapture:O} {s.FrameCount} {s.TotalBytes} {s.Status} {s.VerifiedAt:O} " + + $"{string.Join('|', s.Tags)} {s.SettingsVersion}/{s.FramesVersion} {s.MedianInterval}"); + foreach (var f in await db.Frames.ListBySequenceAsync(s.Id)) + lines.Add($" F {f.Id} {f.Index} {f.RelativePath} {f.OriginalFileName} {f.Size} {f.Sha256} {f.CapturedAt:O} {f.Width}x{f.Height} {f.Format} {f.Flags}"); + } + return string.Join('\n', lines); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs new file mode 100644 index 00000000..71d41e21 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIndexTests.cs @@ -0,0 +1,180 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveIndexTests(TestContext context) +{ + /// Two folders adopted in place and one folder in the layout whose sidecar was lost. + private static void Populate(ArchivePaths paths) + { + Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Harbour sunset"), Start, 6); + Photos.WriteSeries(Path.Combine(paths.Root, "Trips", "Comet"), Start.AddDays(1), 4, firstSeed: 100); + Photos.WriteSeries(Path.Combine(paths.Root, "2026", "2026-09-30 Canon EOS R5", "originals"), Start.AddDays(2), 5, firstSeed: 200); + } + + [Test] + public async Task AdoptionIndexesFoldersInPlaceWithoutTouchingPhotos() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + var photos = Directory.EnumerateFiles(fx.Paths.Root, "*.JPG", SearchOption.AllDirectories).ToDictionary(p => p, File.ReadAllBytes); + + var summary = await fx.ScanAsync(ScanMode.Full); + Assert.Equal(3, summary.Adopted); + Assert.Empty(summary.Failed); + var stats = await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id); + Assert.Equal(3, stats.Sequences); + Assert.Equal(15L, stats.Photos); + var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset"); + Assert.Equal("Harbour sunset", harbour.Title); + var frames = await fx.Db.Frames.ListBySequenceAsync(harbour.Id); + Assert.Equal("Trips/Harbour sunset/IMG_0001.JPG", frames[0].RelativePath); + Assert.Equal(Start, frames[0].CapturedAt); + Assert.Equal(10.0, (await fx.Db.Sequences.GetAsync(harbour.Id))!.MedianInterval); + Assert.FileExists(Path.Combine(fx.Paths.Root, "Trips", "Harbour sunset", ".titano", "sequence.json")); + foreach (var (path, bytes) in photos) Assert.BytesEqual(bytes, File.ReadAllBytes(path)); + } + + [Test] + public async Task AnUnchangedArchiveRescansWithoutReadingSidecars() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var again = await fx.ScanAsync(); + Assert.Equal(3, again.Folders); + Assert.Equal(0, again.Indexed); + + // A photo dropped into a folder and a title edited in a sidecar by another machine. + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Start.AddDays(1).AddMinutes(1), 1, firstSeed: 150, firstNumber: 5); + var sidecar = fx.Paths.Sequence("Trips/Harbour sunset").ManifestPath; + var manifest = ManifestStore.ReadSequence(sidecar); + ManifestStore.WriteSequence(sidecar, manifest with { Title = "Harbour at dusk", Versions = manifest.Versions with { Settings = 5 } }); + + var changed = await fx.ScanAsync(); + Assert.Equal(2, changed.Indexed); + var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + Assert.Equal("Harbour at dusk", locations.Single(l => l.Folder == "Trips/Harbour sunset").Title); + Assert.Equal(5, (await fx.Db.Sequences.GetAsync(locations.Single(l => l.Folder == "Trips/Comet").Id))!.FrameCount); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task RebuildingFromSidecarsGivesTheSameIndex() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(ScanMode.Full); + var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet"); + await fx.Service.SetSequenceInfoAsync(comet.Id, "Comet", ["sky", "night"], 5, "Clear", fx.Ct); + var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); + + await fx.ReopenWithEmptyIndexAsync(); + var rebuilt = await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(0, rebuilt.Adopted); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + + // A second rebuild reads the snapshot written by the first and still yields the same index. + Assert.NotNull(IndexSnapshot.TryLoad(fx.Paths.SnapshotPath)); + await fx.ReopenWithEmptyIndexAsync(); + await fx.Service.Indexer.ScanAsync(fx.Archive, ScanMode.Rebuild, null, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + } + + [Test] + public async Task RebuildDropsRowsWhoseSidecarIsGoneAndScanKeepsThem() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + Directory.Move(Path.Combine(fx.Paths.Root, "Trips", "Comet"), Path.Combine(context.CreateTempDirectory("elsewhere"), "Comet")); + await fx.ScanAsync(); + Assert.Equal(3, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count); + var orphans = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct); + Assert.Equal("Trips/Comet", orphans.MissingSequences.Single().Folder); + + var summary = await fx.ScanAsync(ScanMode.Rebuild); + Assert.Equal(1, summary.Removed); + Assert.Equal(2, (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Count); + } + + [Test] + public async Task ADamagedSidecarIsKeptAsideAndRebuiltWithTheSameIds() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var harbour = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Harbour sunset"); + var frameIds = (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id).ToList(); + var sidecar = fx.Paths.Sequence(harbour.Folder).ManifestPath; + File.WriteAllText(sidecar, "{ \"id\": \"broken"); + + var summary = await fx.ScanAsync(); + Assert.Empty(summary.Failed); + Assert.Equal(harbour.Id, ManifestStore.ReadSequence(sidecar).Id); + Assert.SequenceEqual(frameIds, (await fx.Db.Frames.ListBySequenceAsync(harbour.Id)).Select(f => f.Id)); + Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(sidecar)!, "sequence.json.damaged-*")); + } + + [Test] + public async Task ACopiedSequenceFolderGetsItsOwnIdentity() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var source = Path.Combine(fx.Paths.Root, "Trips", "Comet"); + var copy = Path.Combine(fx.Paths.Root, "Trips", "Comet copy"); + Directory.CreateDirectory(Path.Combine(copy, ".titano")); + foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) + File.Copy(file, Path.Combine(copy, Path.GetRelativePath(source, file))); + + await fx.ScanAsync(); + var locations = await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + Assert.Equal(4, locations.Count); + var original = locations.Single(l => l.Folder == "Trips/Comet"); + var copied = locations.Single(l => l.Folder == "Trips/Comet copy"); + Assert.NotEqual(original.Id, copied.Id); + Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(original.Id)).Count); + Assert.Equal(4, (await fx.Db.Frames.ListBySequenceAsync(copied.Id)).Count); + Assert.Equal(copied.Id, ManifestStore.ReadSequence(fx.Paths.Sequence("Trips/Comet copy").ManifestPath).Id); + } + + [Test] + public async Task AnOfflineArchiveIsLeftAsItWas() + { + var root = context.CreateTempDirectory("archive"); + await using var fx = await CreateAsync(context, root); + Populate(fx.Paths); + await fx.ScanAsync(); + var before = await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id); + var unplugged = root + "-unplugged"; + Directory.Move(root, unplugged); + Assert.Equal(ArchiveAvailability.Offline, ArchiveService.GetAvailability(fx.Archive)); + await Assert.ThrowsAsync(() => fx.ScanAsync(ScanMode.Rebuild)); + Assert.Equal(before, await Photos.DumpIndexAsync(fx.Db, fx.Archive.Id)); + Directory.Move(unplugged, root); + Assert.Equal(ArchiveAvailability.Online, ArchiveService.GetAvailability(fx.Archive)); + } + + [Test] + public async Task SequenceInfoIsWrittenToSidecarAndIndexTogether() + { + await using var fx = await CreateAsync(context); + Populate(fx.Paths); + await fx.ScanAsync(); + var comet = (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).Single(l => l.Folder == "Trips/Comet"); + var updated = await fx.Service.SetSequenceInfoAsync(comet.Id, " Comet 2026 ", ["Night"], 4, "Tripod", fx.Ct); + Assert.Equal("Comet 2026", updated.Title); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(comet.Folder).ManifestPath); + Assert.Equal("Comet 2026", manifest.Title); + Assert.SequenceEqual(["Night"], manifest.Tags); + Assert.Equal(updated.SettingsVersion, manifest.Versions.Settings); + Assert.SequenceEqual(["Night"], (await fx.Db.Sequences.GetAsync(comet.Id))!.Tags); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs new file mode 100644 index 00000000..d9184407 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIngestTests.cs @@ -0,0 +1,124 @@ +using Titano.Core.Archiving; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveIngestTests(TestContext context) +{ + private static IngestRequest Request(ArchiveFixture fx, (string Path, long Size, string Sha256) staged, string? sessionKey = "upload-1", + Guid? sequenceId = null, string? title = null, string name = "IMG_0001.JPG") => + new(fx.Archive.Id, sequenceId, title, sequenceId is null && title is null ? sessionKey : null, staged.Path, name, staged.Size, staged.Sha256, null, null); + + [Test] + public async Task ASessionKeyCreatesOneSequenceNamedByTheTemplates() + { + await using var fx = await CreateAsync(context); + var first = fx.Stage(Start, 1); + var a = await fx.Service.Ingest.CommitAsync(Request(fx, first), fx.Ct); + var b = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(10), 2), name: "IMG_0002.JPG"), fx.Ct); + + Assert.Equal(a.SequenceId, b.SequenceId); + Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-00_0001.JPG", a.RelativePath); + Assert.Equal("2026/2026-09-28 Canon EOS R5/originals/2026-09-28_18-00-10_0002.JPG", b.RelativePath); + Assert.Equal((0, 1), (a.Index, b.Index)); + Assert.False(a.Duplicate); + Assert.FileMissing(first.Path); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("2026/2026-09-28 Canon EOS R5").ManifestPath); + Assert.Equal(a.SequenceId, manifest.Id); + Assert.SequenceEqual(["upload-1"], manifest.SessionKeys); + Assert.Equal("IMG_0002.JPG", manifest.Frames[1].Original); + Assert.Equal(first.Sha256, manifest.Frames[0].Sha256); + Assert.Equal(2, (await fx.Db.Sequences.GetAsync(a.SequenceId))!.FrameCount); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task TheSameContentIsReportedAsDuplicateAndNotCopiedTwice() + { + await using var fx = await CreateAsync(context); + var original = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 7)), fx.Ct); + var again = fx.Stage(Start, 7); + var duplicate = await fx.Service.Ingest.CommitAsync(Request(fx, again), fx.Ct); + Assert.True(duplicate.Duplicate); + Assert.Equal(original.FrameId, duplicate.FrameId); + Assert.Equal(original.RelativePath, duplicate.RelativePath); + Assert.FileExists(again.Path); + Assert.Equal(1, (await fx.Db.Sequences.GetAsync(original.SequenceId))!.FrameCount); + } + + [Test] + public async Task ParallelCommitsToOneSequenceGiveGapFreeIndexesAndAValidSidecar() + { + await using var fx = await CreateAsync(context); + const int count = 40; + var staged = Enumerable.Range(0, count).Select(i => fx.Stage(Start.AddSeconds(10 * i), 1000 + i)).ToList(); + // Arrival order differs from capture order, as with parallel chunked uploads. + var order = Enumerable.Range(0, count).OrderBy(i => (i * 17) % count).ToList(); + var results = await Task.WhenAll(order.Select(i => Task.Run(() => + fx.Service.Ingest.CommitAsync(Request(fx, staged[i], name: $"IMG_{i:0000}.JPG"), fx.Ct)))); + + Assert.Count(1, results.Select(r => r.SequenceId).Distinct()); + Assert.False(results.Any(r => r.Duplicate)); + var sequenceId = results[0].SequenceId; + var row = (await fx.Db.Sequences.GetAsync(sequenceId))!; + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence(row.Folder).ManifestPath); + Assert.SequenceEqual(Enumerable.Range(0, count), manifest.Frames.Select(f => f.I)); + Assert.SequenceEqual(Enumerable.Range(0, count).Select(i => Start.AddSeconds(10 * i)), manifest.Frames.Select(f => f.CapturedAt!.Value)); + Assert.Count(count, manifest.Frames.Select(f => f.File).Distinct()); + Assert.True(manifest.Frames.All(f => File.Exists(fx.Paths.Sequence(row.Folder).File(f.File)))); + var frames = await fx.Db.Frames.ListBySequenceAsync(sequenceId); + Assert.SequenceEqual(manifest.Frames.Select(f => (f.Id, f.I)), frames.Select(f => (f.Id, f.Index))); + Assert.Equal(count, row.FrameCount); + Assert.Empty(Directory.EnumerateFiles(Path.Combine(fx.Paths.StagingDirectory, "uploads"))); + } + + [Test] + public async Task ExplicitTargetsUseTheGivenSequenceOrTitle() + { + await using var fx = await CreateAsync(context); + var titled = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1), title: "Harbour"), fx.Ct); + var sameTitle = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(5), 2), title: "harbour "), fx.Ct); + var byId = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(9), 3), sequenceId: titled.SequenceId), fx.Ct); + Assert.Equal(titled.SequenceId, sameTitle.SequenceId); + Assert.Equal(titled.SequenceId, byId.SequenceId); + Assert.Equal("Harbour", (await fx.Db.Sequences.GetAsync(titled.SequenceId))!.Title); + await Assert.ThrowsAsync(() => fx.Service.Ingest.CommitAsync( + new IngestRequest(fx.Archive.Id, titled.SequenceId, "Both", null, fx.Stage(Start, 4).Path, "x.jpg", 1, "00", null, null), fx.Ct)); + } + + [Test] + public async Task AFrameCapturedEarlierIsInsertedInOrderAndTheIndexFollows() + { + await using var fx = await CreateAsync(context); + var late = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddMinutes(1), 1)), fx.Ct); + var early = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 2)), fx.Ct); + Assert.Equal(0, early.Index); + var frames = await fx.Db.Frames.ListBySequenceAsync(late.SequenceId); + Assert.SequenceEqual([early.FrameId, late.FrameId], frames.Select(f => f.Id)); + Assert.SequenceEqual([0, 1], frames.Select(f => f.Index)); + } + + [Test] + public async Task ABatchInterruptedBetweenMoveAndSidecarIsCompletedOnTheNextCommit() + { + await using var fx = await CreateAsync(context); + var first = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start, 1)), fx.Ct); + var row = (await fx.Db.Sequences.GetAsync(first.SequenceId))!; + var sequence = fx.Paths.Sequence(row.Folder); + // Simulated crash: the journal was written and the file moved, but sequence.json was not rewritten. + var bytes = Photos.Jpeg(Start.AddSeconds(10), 2); + File.WriteAllBytes(sequence.File("originals/moved.JPG"), bytes); + var orphan = new SidecarFrame { Id = Guid.CreateVersion7(), File = "originals/moved.JPG", Size = bytes.Length, CapturedAt = Start.AddSeconds(10) }; + IngestJournal.Write(sequence, new IngestJournal([orphan])); + + var next = await fx.Service.Ingest.CommitAsync(Request(fx, fx.Stage(Start.AddSeconds(20), 3)), fx.Ct); + Assert.Equal(2, next.Index); + var manifest = ManifestStore.ReadSequence(sequence.ManifestPath); + Assert.SequenceEqual([first.FrameId, orphan.Id, next.FrameId], manifest.Frames.Select(f => f.Id)); + Assert.FileMissing(sequence.IngestJournalPath); + Assert.Equal(3, (await fx.Db.Frames.ListBySequenceAsync(first.SequenceId)).Count); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs new file mode 100644 index 00000000..87a0215a --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveIntegrityTests.cs @@ -0,0 +1,160 @@ +using Titano.Core.Archiving; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveIntegrityTests(TestContext context) +{ + private static Task VerifyAsync(ArchiveFixture fx, IReadOnlyList? ids = null) => + fx.Service.Verifier.VerifyAsync(fx.Archive, ids, default, (_, _, _) => ValueTask.CompletedTask, null, fx.Ct); + + private static async Task IngestSeriesAsync(ArchiveFixture fx, string key, int count, int seed) + { + Guid sequence = default; + for (var i = 0; i < count; i++) + { + var staged = fx.Stage(Start.AddSeconds(10 * i), seed + i); + sequence = (await fx.Service.Ingest.CommitAsync(new IngestRequest(fx.Archive.Id, null, null, key, staged.Path, $"IMG_{i:0000}.JPG", + staged.Size, staged.Sha256, null, null), fx.Ct)).SequenceId; + } + return sequence; + } + + [Test] + public async Task VerifyFlagsAFlippedByteAndAMissingFileInSidecarAndIndex() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 5, 1); + var clean = await VerifyAsync(fx); + Assert.Equal(5L, clean.Ok); + Assert.Empty(clean.Problems); + Assert.True((await fx.Db.Sequences.GetAsync(id))!.Status.HasFlag(SequenceStatus.Verified)); + + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + var damaged = folder.File(manifest.Frames[1].File); + var bytes = File.ReadAllBytes(damaged); + bytes[bytes.Length / 3] ^= 0x10; + File.WriteAllBytes(damaged, bytes); + File.Delete(folder.File(manifest.Frames[3].File)); + + var report = await VerifyAsync(fx); + Assert.SequenceEqual([(manifest.Frames[1].Id, FrameHealth.Corrupt), (manifest.Frames[3].Id, FrameHealth.Missing)], + report.Problems.Select(p => (p.FrameId, p.Health))); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + Assert.Equal(FrameFlags.Corrupt, frames[1].Flags); + Assert.Equal(FrameFlags.Missing, frames[3].Flags); + var after = ManifestStore.ReadSequence(folder.ManifestPath); + Assert.Equal(FrameFlags.Corrupt, after.Frames[1].Flags); + Assert.False(after.Status.HasFlag(SequenceStatus.Verified)); + var row = (await fx.Db.Sequences.GetAsync(id))!; + Assert.True(row.Status.HasFlag(SequenceStatus.HasErrors)); + Assert.NotNull(row.VerifiedAt); + var health = (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health; + Assert.Equal((1L, 1L), (health.MissingFrames, health.CorruptFrames)); + } + + [Test] + public async Task VerifyGivesAdoptedPhotosTheirFirstHash() + { + await using var fx = await CreateAsync(context); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "Adopted"), Start, 3); + await fx.ScanAsync(); + var report = await VerifyAsync(fx); + Assert.Equal(3L, report.Hashed); + var manifest = ManifestStore.ReadSequence(fx.Paths.Sequence("Adopted").ManifestPath); + Assert.True(manifest.Frames.All(f => f.Sha256 is { Length: 64 })); + var frames = await fx.Db.Frames.ListBySequenceAsync(manifest.Id); + Assert.SequenceEqual(manifest.Frames.Select(f => f.Sha256), frames.Select(f => f.Sha256)); + Assert.Equal(3L, (await VerifyAsync(fx)).Ok); + } + + [Test] + public async Task DuplicatesAcrossSequencesAreFoundAndFlaggedAfterADryRun() + { + await using var fx = await CreateAsync(context); + var a = await IngestSeriesAsync(fx, "a", 3, 1); + var b = await IngestSeriesAsync(fx, "b", 2, 1); // the same two photos again, in another sequence + var groups = await fx.Service.FindDuplicatesAsync(fx.Archive.Id, fx.Ct); + Assert.Equal(2, groups.Count); + Assert.True(groups.All(g => g.Frames.Count == 2 && g.Frames[0].SequenceId == a && g.Frames[1].SequenceId == b)); + + var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(2, dry.DuplicateFlagChanges); + Assert.Equal(0L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames); + + var run = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(2, run.DuplicateFlagChanges); + Assert.Equal(2L, (await fx.Db.Sequences.GetStatsAsync(fx.Archive.Id)).Health.DuplicateFrames); + var folder = (await fx.Db.Sequences.GetAsync(b))!.Folder; + Assert.True(ManifestStore.ReadSequence(fx.Paths.Sequence(folder).ManifestPath).Frames.All(f => f.Flags.HasFlag(FrameFlags.Duplicate))); + Assert.Equal(0, (await fx.Service.Maintenance.RunAsync(fx.Archive, false, 0, _ => ValueTask.CompletedTask, fx.Ct)).DuplicateFlagChanges); + } + + [Test] + public async Task OrphansListUnindexedAndMissingFilesAndTheFixIndexesThem() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 3, 1); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var manifest = ManifestStore.ReadSequence(folder.ManifestPath); + File.Delete(folder.File(manifest.Frames[0].File)); + File.WriteAllBytes(Path.Combine(folder.Originals, "dropped.JPG"), Photos.Jpeg(Start.AddMinutes(5), 50)); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "New folder"), Start, 1, firstSeed: 60); + + var report = await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct); + var orphans = report.Sequences.Single(); + Assert.SequenceEqual(["originals/dropped.JPG"], orphans.UnindexedFiles); + Assert.Equal(manifest.Frames[0].Id, orphans.MissingFiles.Single().Id); + Assert.SequenceEqual(["New folder"], report.UnindexedFolders); + + await fx.Service.ReconcileAsync(id, fx.Ct); + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + Assert.Equal(4, frames.Count); + Assert.Equal(FrameFlags.Missing, frames[0].Flags); + Assert.Equal("dropped.JPG", frames[^1].FileName); + Assert.Empty((await fx.Service.FindOrphansAsync(fx.Archive.Id, fx.Ct)).Sequences.Where(s => s.UnindexedFiles.Count > 0)); + } + + [Test] + public async Task MaintenanceRemovesOnlyStaleLeftovers() + { + await using var fx = await CreateAsync(context); + var id = await IngestSeriesAsync(fx, "a", 2, 1); + var folder = fx.Paths.Sequence((await fx.Db.Sequences.GetAsync(id))!.Folder); + var oldTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.1234abcd.tmp"); + var freshTemp = Path.Combine(folder.MetaDirectory, ".sequence.json.5678abcd.tmp"); + var oldStaged = Path.Combine(fx.Paths.StagingDirectory, "old.part"); + File.WriteAllText(oldTemp, "x"); + File.WriteAllText(freshTemp, "x"); + File.WriteAllText(oldStaged, "x"); + File.SetLastWriteTimeUtc(oldTemp, DateTime.UtcNow.AddHours(-2)); + File.SetLastWriteTimeUtc(oldStaged, DateTime.UtcNow.AddDays(-2)); + var past = DateTimeOffset.UtcNow.AddHours(-1); + File.WriteAllText(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"), + $$"""{"token":"t","owner":"crashed","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past.AddMinutes(2):O}}"}"""); + + var dry = await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: true, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.Equal(2, dry.TemporaryFiles.Count); + Assert.SequenceEqual(["seq-dead"], dry.StaleLeases); + Assert.FileExists(oldTemp); + + await fx.Service.Maintenance.RunAsync(fx.Archive, dryRun: false, 0, _ => ValueTask.CompletedTask, fx.Ct); + Assert.FileMissing(oldTemp); + Assert.FileMissing(oldStaged); + Assert.FileExists(freshTemp); + Assert.FileMissing(Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease")); + Assert.True(folder.Frames().All(File.Exists)); + } +} + +internal static class SequencePathsExtensions +{ + /// Absolute paths of every frame listed in the sidecar. + public static IEnumerable Frames(this SequencePaths sequence) => + ManifestStore.ReadSequence(sequence.ManifestPath).Frames.Select(f => sequence.File(f.File)); +} diff --git a/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs new file mode 100644 index 00000000..91b954e3 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ArchiveJobTests.cs @@ -0,0 +1,130 @@ +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Archiving.Jobs; +using Titano.Data.Repositories; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +/// Every archive job handler run through the engine, resuming from a checkpoint a crashed attempt left. +[Category("archive")] +public sealed class ArchiveJobTests(TestContext context) +{ + private static void Populate(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "a"), Start, 6); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "b"), Start.AddDays(1), 6, firstSeed: 50); + } + + private static async Task> LocationsAsync(ArchiveFixture fx) => + await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id); + + [Test] + public async Task ScanAndRebuildJobsRunAndAScanResumesAfterItsCheckpoint() + { + await using var fx = await CreateAsync(context); + Populate(fx); + var engine = await fx.StartEngineAsync(); + var resumed = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Scan(fx.Archive), "after:a"); + await fx.WaitForAsync(resumed.Id, JobState.Done); + Assert.SequenceEqual(["b"], (await LocationsAsync(fx)).Select(l => l.Folder)); + + var full = await engine.EnqueueAsync(ArchiveJobs.Scan(fx.Archive, full: true), cancellationToken: fx.Ct); + var done = await fx.WaitForAsync(full.Id, JobState.Done); + Assert.Contains("2 folders", done.LogTail); + Assert.SequenceEqual(["a", "b"], (await LocationsAsync(fx)).Select(l => l.Folder)); + + var rebuild = await engine.EnqueueAsync(ArchiveJobs.Rebuild(fx.Archive), cancellationToken: fx.Ct); + await fx.WaitForAsync(rebuild.Id, JobState.Done); + Assert.Equal(2, (await LocationsAsync(fx)).Count); + } + + [Test] + public async Task AVerifyJobResumesAtTheNextSequence() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await fx.ScanAsync(); + var engine = await fx.StartEngineAsync(); + var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Verify(fx.Archive), "1:2"); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + var (a, b) = ((await LocationsAsync(fx))[0], (await LocationsAsync(fx))[1]); + Assert.Null((await fx.Db.Sequences.GetAsync(a.Id))!.VerifiedAt); + Assert.NotNull((await fx.Db.Sequences.GetAsync(b.Id))!.VerifiedAt); + var hashes = (await fx.Db.Frames.ListBySequenceAsync(b.Id)).Select(f => f.Sha256).ToList(); + Assert.True(hashes.Take(2).All(h => h is null) && hashes.Skip(2).All(h => h is not null)); + Assert.Contains("4 photos checked", done.LogTail); + } + + [Test] + public async Task ThumbnailAndProxyJobsResumeInsideASequence() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await fx.ScanAsync(); + var a = (await LocationsAsync(fx))[0]; + var engine = await fx.StartEngineAsync(); + var thumbs = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Thumbnails([a.Id]), "0:3"); + await fx.WaitForAsync(thumbs.Id, JobState.Done); + var proxies = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Proxies([a.Id]), "0:3"); + await fx.WaitForAsync(proxies.Id, JobState.Done); + + var sequence = fx.Paths.Sequence(a.Folder); + using (var pack = ThumbPackReader.Open(sequence.ThumbsPackPath)) + { + Assert.SequenceEqual([3, 4, 5], pack.Entries.Keys.Order()); + Assert.NotNull(pack.Read(4)); + } + Assert.SequenceEqual(["0004.jpg", "0005.jpg", "0006.jpg"], Directory.GetFiles(sequence.Proxies).Select(Path.GetFileName).Order()); + var status = (await fx.Db.Sequences.GetAsync(a.Id))!.Status; + Assert.True(status.HasFlag(SequenceStatus.ThumbsBuilt) && status.HasFlag(SequenceStatus.ProxiesBuilt)); + + // A full pass reuses what is current and builds the rest. + var all = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([a.Id]), cancellationToken: fx.Ct); + var done = await fx.WaitForAsync(all.Id, JobState.Done); + Assert.Contains("3 made, 3 reused", done.LogTail); + } + + [Test] + public async Task AMaintenanceJobResumesAfterItsDoneSteps() + { + await using var fx = await CreateAsync(context); + Populate(fx); + await fx.ScanAsync(); + Directory.CreateDirectory(fx.Paths.LocksDirectory); + var past = DateTimeOffset.UtcNow.AddHours(-1); + var lease = Path.Combine(fx.Paths.LocksDirectory, "seq-dead.lease"); + File.WriteAllText(lease, $$"""{"token":"t","owner":"x","host":"h","processId":1,"acquired":"{{past:O}}","expires":"{{past:O}}"}"""); + var temp = Path.Combine(fx.Paths.MetaDirectory, ".archive.json.0badf00d.tmp"); + File.WriteAllText(temp, "x"); + File.SetLastWriteTimeUtc(temp, DateTime.UtcNow.AddHours(-3)); + + var engine = await fx.StartEngineAsync(); + var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Maintenance(fx.Archive), "1"); + await fx.WaitForAsync(job.Id, JobState.Done); + Assert.FileExists(lease); + Assert.FileMissing(temp); + } + + [Test] + public async Task AnImportJobResumesAtItsCheckpoint() + { + await using var fx = await CreateAsync(context); + var card = context.CreateTempDirectory("card"); + Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 40); + var media = await fx.Service.Imports.RegisterCardAsync(new CardVolume("1-A-1", 1, "A", 1), fx.Ct); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + var engine = await fx.StartEngineAsync(); + var job = await fx.EnqueueWithCheckpointAsync(engine, ArchiveJobs.Import(fx.Archive, plan, media.Id), "32"); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Equal("40", done.Checkpoint); + var id = (await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, plan.Sessions[0].SessionKey))!.Value; + var frames = await fx.Db.Frames.ListBySequenceAsync(id); + Assert.SequenceEqual(plan.Items.Skip(32).Select(i => i.FileName), frames.Select(f => f.FileName)); + Assert.Equal(8, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); + Assert.Contains("8 copied", done.LogTail); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs b/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs new file mode 100644 index 00000000..94c7e572 --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/ImportServiceTests.cs @@ -0,0 +1,135 @@ +using Titano.Core.Archiving; +using Titano.Core.Import; +using Titano.Core.Model; +using Titano.Data.Archiving; +using Titano.Data.Archiving.Jobs; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("import")] +public sealed class ImportServiceTests(TestContext context) +{ + private static readonly CardVolume Volume = new("3F2A9C11-EOS_DIGITAL-127865454592", 0x3F2A9C11, "EOS_DIGITAL", 127_865_454_592); + + private string Card(int frames) + { + var card = context.CreateTempDirectory("card"); + Photos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames); + return card; + } + + private static async Task> FramesAsync(ArchiveFixture fx, PlannedSession session) + { + var id = await fx.Db.Sequences.FindBySessionKeyAsync(fx.Archive.Id, session.SessionKey); + Assert.NotNull(id); + return [.. await fx.Db.Frames.ListBySequenceAsync(id.Value)]; + } + + [Test] + public async Task AnImportJobCopiesThePlanAndRemembersTheCard() + { + await using var fx = await CreateAsync(context); + var card = Card(12); + var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); + Assert.Null(media.Nickname); + Assert.Equal(media.Id, (await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct)).Id); + await fx.Service.Imports.SaveCardAsync(media.Id, "EOS R5 card A", new CardProfile { SessionGapMinutes = 30 }, autoImport: true, fx.Ct); + + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + var engine = await fx.StartEngineAsync(); + var job = await engine.EnqueueAsync(ArchiveJobs.Import(fx.Archive, plan, media.Id), cancellationToken: fx.Ct); + var done = await fx.WaitForAsync(job.Id, JobState.Done); + Assert.Equal(plan.CopyBytes, done.ProgressDone); + + var session = plan.Sessions.Single(); + var frames = await FramesAsync(fx, session); + Assert.SequenceEqual(session.Items.Select(i => $"{session.Folder}/originals/{i.FileName}"), frames.Select(f => f.RelativePath)); + Assert.SequenceEqual(session.Items.Select(i => Path.GetFileName(i.SourcePath)), frames.Select(f => f.OriginalFileName!)); + foreach (var (item, frame) in session.Items.Zip(frames)) + Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)), File.ReadAllBytes(fx.Paths.FromRelative(frame.RelativePath))); + var remembered = (await fx.Db.MediaCards.GetAsync(media.Id))!; + Assert.Equal("EOS R5 card A", remembered.Nickname); + Assert.Equal("Canon EOS R5", remembered.CameraModel); + Assert.Equal(30, ImportService.ProfileOf(remembered).SessionGapMinutes); + Assert.Equal(12, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); + + var again = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + Assert.True(again.Items.All(i => i.Action == ImportAction.SkipImported)); + Assert.Equal(session.Folder, again.Sessions[0].Folder); + var elsewhere = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct); + Assert.True(elsewhere.Items.All(i => i.Action == ImportAction.SkipInArchive && i.ExistingFrameId is not null)); + } + + [Test] + public async Task ACardPulledMidImportResumesWithoutGapsOrDoubles() + { + await using var fx = await CreateAsync(context); + var card = Card(70); + var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + var parameters = new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null); + var jobId = Guid.CreateVersion7(); + var checkpoint = 0; + await Assert.ThrowsAsync(() => fx.Service.Imports.ExecuteAsync(parameters, jobId, 0, batch => + { + checkpoint = batch.NextItem; + if (Directory.Exists(card)) Directory.Move(card, card + "-out"); + return ValueTask.CompletedTask; + }, null, fx.Ct)); + Assert.True(checkpoint is > 0 and < 70); + Directory.Move(card + "-out", card); + + var final = await fx.Service.Imports.ExecuteAsync(parameters, jobId, checkpoint, _ => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal(70 - checkpoint, final.Copied); + var frames = await FramesAsync(fx, plan.Sessions.Single()); + Assert.SequenceEqual(Enumerable.Range(0, 70), frames.Select(f => f.Index)); + Assert.SequenceEqual(plan.Items.Select(i => i.FileName), frames.Select(f => f.FileName)); + Assert.Equal(70, (await fx.Db.MediaCards.GetImportedKeysAsync(media.Id)).Count); + Assert.Empty(Directory.EnumerateFiles(fx.Paths.StagingDirectory)); + } + + [Test] + public async Task RepeatingAnImportAfterACrashCommitsNothingTwice() + { + await using var fx = await CreateAsync(context); + var card = Card(20); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), null, null, fx.Ct); + var parameters = new ImportJobParameters(fx.Archive.Id, plan, null, true, null); + await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); + // The checkpoint of the last batch was lost: the run repeats from the start. + var repeat = await fx.Service.Imports.ExecuteAsync(parameters, Guid.NewGuid(), 0, _ => ValueTask.CompletedTask, null, fx.Ct); + Assert.Equal(0, repeat.Copied); + Assert.Equal(20, repeat.Skipped); + Assert.Equal(20, (await FramesAsync(fx, plan.Sessions.Single())).Count); + } + + [Test] + public async Task ErasingTheCardDeletesOnlyVerifiedImports() + { + await using var fx = await CreateAsync(context); + var card = Card(6); + File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "keep me"); + var media = await fx.Service.Imports.RegisterCardAsync(Volume, fx.Ct); + var plan = await fx.Service.PlanImportAsync(fx.Archive.Id, card, new ImportOptions(), media.Id, null, fx.Ct); + await fx.Service.Imports.ExecuteAsync(new ImportJobParameters(fx.Archive.Id, plan, media.Id, true, null), Guid.NewGuid(), 0, + _ => ValueTask.CompletedTask, null, fx.Ct); + var frame = (await FramesAsync(fx, plan.Sessions.Single()))[2]; + var copy = fx.Paths.FromRelative(frame.RelativePath); + var bytes = File.ReadAllBytes(copy); + bytes[^20] ^= 0x01; + File.WriteAllBytes(copy, bytes); + + var erase = await fx.Service.PlanCardEraseAsync(fx.Archive.Id, media.Id, card, null, fx.Ct); + Assert.Equal(5, erase.ToDelete.Count()); + Assert.Equal(EraseVerdict.CopyMismatch, erase.Items.Single(i => i.Candidate.SourcePath == plan.Items.ElementAt(2).SourcePath).Verdict); + Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count()); + + var result = CardEraser.Execute(erase); + Assert.Equal(5, result.Deleted); + Assert.SequenceEqual([plan.Items.ElementAt(2).SourcePath], + Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/'))); + Assert.FileExists(Path.Combine(card, "DCIM", "100CANON", "notes.txt")); + } +} diff --git a/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs new file mode 100644 index 00000000..5cf3130a --- /dev/null +++ b/tests/Titano.Data.Tests/Archiving/MassRenameTests.cs @@ -0,0 +1,113 @@ +using Titano.Core.Archiving; +using Titano.Data.Archiving; +using Titano.Testing; +using static Titano.Data.Tests.Archiving.ArchiveFixture; + +namespace Titano.Data.Tests.Archiving; + +[Category("archive")] +public sealed class MassRenameTests(TestContext context) +{ + private static async Task> PopulateAsync(ArchiveFixture fx) + { + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "a"), Start, 3); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "b"), Start.AddDays(1), 3, firstSeed: 10); + Photos.WriteSeries(Path.Combine(fx.Paths.Root, "inbox", "c"), Start.AddDays(2), 3, firstSeed: 20); + await fx.ScanAsync(); + return (await fx.Db.Sequences.ListLocationsAsync(fx.Archive.Id)).OrderBy(l => l.Folder).Select(l => l.Id).ToList(); + } + + /// Folder, title and every frame path of each sequence as the index and the disk see them. + private static async Task> StateAsync(ArchiveFixture fx, IEnumerable ids) + { + var state = new List(); + foreach (var id in ids) + { + var s = (await fx.Db.Sequences.GetAsync(id))!; + var sidecar = ManifestStore.ReadSequence(fx.Paths.Sequence(s.Folder).ManifestPath); + Assert.Equal(id, sidecar.Id); + Assert.Equal(s.Title, sidecar.Title); + foreach (var f in await fx.Db.Frames.ListBySequenceAsync(id)) Assert.FileExists(fx.Paths.FromRelative(f.RelativePath)); + state.Add($"{s.Folder}|{s.Title}|{string.Join(',', (await fx.Db.Frames.ListBySequenceAsync(id)).Select(f => f.RelativePath))}"); + } + return state; + } + + private static List Expected(RenamePlan plan, IEnumerable before) => + before.Select(line => + { + var folder = line.Split('|')[0]; + var step = plan.Steps.SingleOrDefault(s => s.FromFolder == folder); + return step is null ? line : $"{step.ToFolder}|{step.ToTitle}|{line.Split('|')[2].Replace(step.FromFolder + "/", step.ToFolder + "/")}"; + }).ToList(); + + [Test] + public async Task ApplyingAPlanDoesExactlyWhatTheDryRunSaid() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "{year}/{date} {title}", "Trip", fx.Ct); + Assert.SequenceEqual(["2026/2026-09-28 Trip", "2026/2026-09-29 Trip", "2026/2026-09-30 Trip"], plan.Steps.Select(s => s.ToFolder)); + Assert.SequenceEqual(before, await StateAsync(fx, ids)); + + await fx.Service.ApplyRenameAsync(plan, null, fx.Ct); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.False(Directory.Exists(Path.Combine(fx.Paths.Root, "inbox", "a"))); + Assert.Equal(0, (await fx.ScanAsync()).Indexed); + } + + [Test] + public async Task AnInterruptedRenameResumesFromItsJournal() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", null, fx.Ct); + using var cts = new CancellationTokenSource(); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token)); + var journal = RenameJournal.ListPending(fx.Paths).Single(); + Assert.Count(1, journal.Completed); + + // Crash in the middle of the next step: its folder moved, the index not yet updated. + var next = plan.Steps[1]; + Directory.CreateDirectory(Path.GetDirectoryName(fx.Paths.FromRelative(next.ToFolder))!); + Directory.Move(fx.Paths.FromRelative(next.FromFolder), fx.Paths.FromRelative(next.ToFolder)); + + await fx.Service.OpenAsync(fx.Archive.RootPath, fx.Ct); // the next start finds the journal + Assert.Empty(RenameJournal.ListPending(fx.Paths)); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + } + + [Test] + public async Task AResumedRenameOnTheSameIndexMatchesThePlan() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var before = await StateAsync(fx, ids); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, ids, "renamed/{date}", "{title} 2026", fx.Ct); + using var cts = new CancellationTokenSource(); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, new SyncProgress(_ => cts.Cancel()), cts.Token)); + Assert.Equal(1, await fx.Service.Renamer.ResumePendingAsync(fx.Archive, fx.Ct)); + Assert.SequenceEqual(Expected(plan, before), await StateAsync(fx, ids)); + } + + [Test] + public async Task AStalePlanIsRefused() + { + await using var fx = await CreateAsync(context); + var ids = await PopulateAsync(fx); + var plan = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "x/{date}", null, fx.Ct); + var other = await fx.Service.PlanRenameAsync(fx.Archive.Id, [ids[0]], "y/{date}", null, fx.Ct); + await fx.Service.ApplyRenameAsync(other, null, fx.Ct); + await Assert.ThrowsAsync(() => fx.Service.ApplyRenameAsync(plan, null, fx.Ct)); + Assert.True(Directory.Exists(fx.Paths.FromRelative(other.Steps[0].ToFolder))); + } + + /// Reports synchronously (Progress<T> would post to the thread pool). + private sealed class SyncProgress(Action report) : IProgress + { + public void Report(int value) => report(value); + } +}