diff --git a/i18n/en/Archive.resx b/i18n/en/Archive.resx index 20ad648e..a5952f47 100644 --- a/i18n/en/Archive.resx +++ b/i18n/en/Archive.resx @@ -5,4 +5,26 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 No archive yet + Another Titano ({0}) is writing here. Try again shortly.long: error sentence + The copy of {0} does not match the original.long: error sentence + {0} could not be added to the sequence.long: error sentence + {0} was written by a newer Titano (format {1}).long: error sentence + The archive {0} is offline.long: error sentence + The archive {0} is read-only.long: error sentence + {0} changed after the plan was made.long: error sentence + The card or folder {0} is no longer available.long: error sentence + The archive changed after this plan was made.long: error sentence + Dry run: nothing was changed.long: job log line + Not indexed: {0} ({1})long: job log line + Changed: {0}long: job log line + Damaged: {0}long: job log line + Missing: {0}long: job log line + {0}: {1} made, {2} reused, {3} unreadablelong: job log line + {0} copied, {1} skipped, {2} failedlong: job log line + Not copied: {0} ({1})long: job log line + {0} stale locks, {1} temporary files, {2} packs compacted, {3} duplicate flags changedlong: job log line + {0} unindexed photos, {1} missing photoslong: job log line + {0} folders, {1} indexed, {2} adopted, {3} removedlong: job log line + Sequence not found: {0}long: job log line + {0} photos checked, {1} intact, {2} hashed, {3} with problemslong: job log line diff --git a/i18n/it/Archive.it.resx b/i18n/it/Archive.it.resx index 00b325d1..62523790 100644 --- a/i18n/it/Archive.it.resx +++ b/i18n/it/Archive.it.resx @@ -5,4 +5,26 @@ System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 Nessun archivio + Un altro Titano ({0}) sta scrivendo qui. Riprova tra poco. + La copia di {0} non corrisponde all'originale. + Impossibile aggiungere {0} alla sequenza. + {0} è stato scritto da un Titano più recente (formato {1}). + L'archivio {0} non è raggiungibile. + L'archivio {0} è di sola lettura. + {0} è cambiato dopo la creazione del piano. + La scheda o cartella {0} non è più disponibile. + L'archivio è cambiato dopo la creazione del piano. + Prova: nulla è stato modificato. + Non indicizzata: {0} ({1}) + Modificato: {0} + Danneggiato: {0} + Mancante: {0} + {0}: {1} create, {2} riusate, {3} illeggibili + {0} copiate, {1} saltate, {2} non riuscite + Non copiata: {0} ({1}) + {0} blocchi scaduti, {1} file temporanei, {2} pacchetti compattati, {3} segni di duplicato cambiati + {0} foto non indicizzate, {1} foto mancanti + {0} cartelle, {1} indicizzate, {2} adottate, {3} rimosse + Sequenza non trovata: {0} + {0} foto controllate, {1} integre, {2} con impronta nuova, {3} con problemi diff --git a/src/Titano.Core/Archiving/ArchiveJson.cs b/src/Titano.Core/Archiving/ArchiveJson.cs new file mode 100644 index 00000000..6a893b97 --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveJson.cs @@ -0,0 +1,46 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Titano.Core.Archiving; + +/// Source-generated metadata for every JSON file of the archive layout. +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = true, + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip)] +[JsonSerializable(typeof(ArchiveManifest))] +[JsonSerializable(typeof(SequenceManifest))] +[JsonSerializable(typeof(LeaseRecord))] +[JsonSerializable(typeof(IngestJournal))] +[JsonSerializable(typeof(RenameJournal))] +internal sealed partial class ArchiveJson : JsonSerializerContext; + +/// Indented output for the small files people may open (archive.json, rename journals). +[JsonSourceGenerationOptions( + WriteIndented = true, + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] +[JsonSerializable(typeof(ArchiveManifest))] +[JsonSerializable(typeof(RenameJournal))] +internal sealed partial class ArchiveJsonIndented : JsonSerializerContext; + +/// +/// Writes enum names and reads unknown names (written by a newer build) or numbers as the default value instead of +/// failing, so a sidecar from a newer Titano is still readable. +/// +public sealed class TolerantEnumConverter : JsonConverter + where TEnum : struct, Enum +{ + public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.String && Enum.TryParse(reader.GetString(), ignoreCase: true, out var value) + && Enum.IsDefined(value)) + return value; + if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) reader.Skip(); + return default; + } + + public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToString()); +} diff --git a/src/Titano.Core/Archiving/ArchiveLayout.cs b/src/Titano.Core/Archiving/ArchiveLayout.cs new file mode 100644 index 00000000..ab51614d --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveLayout.cs @@ -0,0 +1,104 @@ +using System.Globalization; +using Titano.Core.IO; + +namespace Titano.Core.Archiving; + +/// Names of the fixed files and folders of the archive layout (BRIEF §7). +public static class ArchiveLayout +{ + /// Metadata folder at the archive root and inside every sequence folder. + public const string MetaFolder = ".titano"; + + public const string ArchiveManifestFile = "archive.json"; + public const string IndexSnapshotFile = "index-snapshot.bin"; + public const string LocksFolder = "locks"; + + /// Verified copies waiting to be committed; on the archive volume so commits are renames. + public const string StagingFolder = "staging"; + + /// Resumable plans of mass operations (rename). + public const string JournalFolder = "journal"; + + public const string OriginalsFolder = "originals"; + public const string ProxiesFolder = "proxies"; + public const string ExportsFolder = "exports"; + public const string SequenceManifestFile = "sequence.json"; + public const string AnalysisFile = "analysis.json"; + public const string ProjectFile = "project.json"; + public const string ThumbsPackFile = "thumbs.pack"; + + /// Moves of an ingest batch that were started but not yet recorded in . + public const string IngestJournalFile = "ingest.pending.json"; + + /// Folder names inside a sequence folder that never hold a sequence of their own. + public static bool IsReservedFolder(string name) => + name.Equals(MetaFolder, StringComparison.OrdinalIgnoreCase) + || name.Equals(OriginalsFolder, StringComparison.OrdinalIgnoreCase) + || name.Equals(ProxiesFolder, StringComparison.OrdinalIgnoreCase) + || name.Equals(ExportsFolder, StringComparison.OrdinalIgnoreCase); + + /// 0001.jpg for frame index 0: proxies are numbered from 1 like the frames of a render. + public static string ProxyFileName(int index) => (index + 1).ToString("0000", CultureInfo.InvariantCulture) + ".jpg"; +} + +/// Absolute paths of one archive root. +public sealed class ArchivePaths +{ + public ArchivePaths(string root) + { + ArgumentException.ThrowIfNullOrWhiteSpace(root); + Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root))); + MetaDirectory = Path.Combine(Root, ArchiveLayout.MetaFolder); + } + + public string Root { get; } + public string MetaDirectory { get; } + public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.ArchiveManifestFile); + public string SnapshotPath => Path.Combine(MetaDirectory, ArchiveLayout.IndexSnapshotFile); + public string LocksDirectory => Path.Combine(MetaDirectory, ArchiveLayout.LocksFolder); + public string StagingDirectory => Path.Combine(MetaDirectory, ArchiveLayout.StagingFolder); + public string JournalDirectory => Path.Combine(MetaDirectory, ArchiveLayout.JournalFolder); + + /// Paths of the sequence stored in (forward slashes). + public SequencePaths Sequence(string relativeFolder) => new(this, relativeFolder); + + /// Archive-relative path with forward slashes, as stored in the index. + public string ToRelative(string path) => LongPath.ToPortableRelative(Root, path); + + public string FromRelative(string relative) => LongPath.FromPortableRelative(Root, relative); +} + +/// Absolute paths of one sequence folder. +public sealed class SequencePaths +{ + internal SequencePaths(ArchivePaths archive, string relativeFolder) + { + ArgumentNullException.ThrowIfNull(relativeFolder); + RelativeFolder = relativeFolder.Replace('\\', '/').Trim('/'); + Folder = archive.FromRelative(RelativeFolder); + MetaDirectory = Path.Combine(Folder, ArchiveLayout.MetaFolder); + } + + /// Folder relative to the archive root, forward slashes; empty when the root itself holds the photos (adopted folder). + public string RelativeFolder { get; } + + public string Folder { get; } + public string MetaDirectory { get; } + public string Originals => Path.Combine(Folder, ArchiveLayout.OriginalsFolder); + public string Proxies => Path.Combine(Folder, ArchiveLayout.ProxiesFolder); + public string Exports => Path.Combine(Folder, ArchiveLayout.ExportsFolder); + public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.SequenceManifestFile); + public string AnalysisPath => Path.Combine(MetaDirectory, ArchiveLayout.AnalysisFile); + public string ProjectPath => Path.Combine(MetaDirectory, ArchiveLayout.ProjectFile); + public string ThumbsPackPath => Path.Combine(MetaDirectory, ArchiveLayout.ThumbsPackFile); + public string IngestJournalPath => Path.Combine(MetaDirectory, ArchiveLayout.IngestJournalFile); + + public string Proxy(int index) => Path.Combine(Proxies, ArchiveLayout.ProxyFileName(index)); + + /// Absolute path of a frame file given its sidecar path (relative to the sequence folder). + public string File(string sequenceRelative) => LongPath.FromPortableRelative(Folder, sequenceRelative); + + /// Archive-relative path of a frame file (as stored in the frames table). + public string ArchiveRelative(string sequenceRelative) => + RelativeFolder.Length == 0 ? sequenceRelative : RelativeFolder + "/" + sequenceRelative; +} diff --git a/src/Titano.Core/Archiving/ArchiveManifest.cs b/src/Titano.Core/Archiving/ArchiveManifest.cs new file mode 100644 index 00000000..6918ff9f --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveManifest.cs @@ -0,0 +1,49 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Naming; + +namespace Titano.Core.Archiving; + +/// +/// .titano/archive.json at the archive root (BRIEF §7). Fields this build does not know are kept in +/// and written back unchanged. +/// +public sealed record ArchiveManifest +{ + public const int CurrentSchemaVersion = 1; + + public int SchemaVersion { get; init; } = CurrentSchemaVersion; + public required Guid Id { get; init; } + public required string Name { get; init; } + public DateTimeOffset Created { get; init; } + public LayoutTemplates LayoutTemplates { get; init; } = new(); + public ProxyPolicy ProxyPolicy { get; init; } = new(); + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// Naming templates used when frames enter the archive (BRIEF §7.1). +public sealed record LayoutTemplates +{ + public string Folder { get; init; } = TemplateDefaults.Folder; + public string File { get; init; } = TemplateDefaults.File; + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// Proxy generation policy (BRIEF §8.5). +public sealed record ProxyPolicy +{ + public const int Disabled = -1; + public const int FullSize = 0; + + /// Long edge in pixels: 1280, 1920 (default) or 2560; or . + public int LongEdge { get; init; } = 1920; + + public int Quality { get; init; } = 92; + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} diff --git a/src/Titano.Core/Archiving/ArchiveScanner.cs b/src/Titano.Core/Archiving/ArchiveScanner.cs new file mode 100644 index 00000000..47f20980 --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveScanner.cs @@ -0,0 +1,77 @@ +using Titano.Core.Imaging; + +namespace Titano.Core.Archiving; + +/// A folder that holds a sequence: it has a sidecar, or photos to adopt. +/// Archive-relative, forward slashes; empty for the root. +/// Stamp of .titano/sequence.json, null when it has none yet. +/// Stamp of the photo directory (its time changes when files are added or removed). +public sealed record SequenceFolder(string RelativeFolder, FileStamp? Sidecar, FileStamp? Photos); + +/// +/// Walks an archive tree (BRIEF §7). No file-system watcher is involved: callers compare the stamps with those of +/// the previous scan. Hidden folders, symbolic links and the reserved sequence subfolders are not entered. +/// +public static class ArchiveScanner +{ + private const int MaxDepth = 16; + + private static readonly EnumerationOptions Listing = new() + { + IgnoreInaccessible = true, + AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System, + RecurseSubdirectories = false, + }; + + /// Every sequence folder under the root, ordered by relative path (ordinal) so job checkpoints are stable. + public static IReadOnlyList FindSequenceFolders(ArchivePaths archive, CancellationToken cancellationToken = default) + { + var found = new List(); + var pending = new Stack<(string Path, int Depth)>(); + pending.Push((archive.Root, 0)); + while (pending.Count > 0) + { + cancellationToken.ThrowIfCancellationRequested(); + var (path, depth) = pending.Pop(); + var relative = archive.ToRelative(path); + var sequence = archive.Sequence(relative == "." ? string.Empty : relative); + var sidecar = FileStamp.Of(sequence.ManifestPath); + var photoDirectory = PhotoDirectory(sequence); + if (sidecar is not null || EnumeratePhotos(photoDirectory).Any()) + found.Add(new SequenceFolder(sequence.RelativeFolder, sidecar, FileStamp.Of(photoDirectory))); + if (depth >= MaxDepth) continue; + foreach (var child in Directory.EnumerateDirectories(path, "*", Listing)) + { + var name = Path.GetFileName(child); + if (!IsHidden(child, name) && !ArchiveLayout.IsReservedFolder(name)) pending.Push((child, depth + 1)); + } + } + found.Sort((a, b) => string.CompareOrdinal(a.RelativeFolder, b.RelativeFolder)); + return found; + } + + /// originals/ when the folder follows the layout, otherwise the folder itself (adopted in place). + public static string PhotoDirectory(SequencePaths sequence) => Directory.Exists(sequence.Originals) ? sequence.Originals : sequence.Folder; + + /// Photos of a sequence folder as sidecar paths (relative to the sequence folder), ordinal order. + public static IReadOnlyList ListPhotos(SequencePaths sequence) + { + var directory = PhotoDirectory(sequence); + return EnumeratePhotos(directory) + .Select(path => Path.GetRelativePath(sequence.Folder, path).Replace('\\', '/')) + .Order(StringComparer.Ordinal) + .ToList(); + } + + /// True for files Titano imports as photos (AppleDouble ._ companions excluded). + public static bool IsPhoto(string fileName) => + !fileName.StartsWith("._", StringComparison.Ordinal) && ImageFormatInfo.IsSupportedExtension(fileName); + + private static IEnumerable EnumeratePhotos(string directory) => + Directory.Exists(directory) + ? Directory.EnumerateFiles(directory, "*", Listing).Where(f => IsPhoto(Path.GetFileName(f)) && !IsHidden(f, Path.GetFileName(f))) + : []; + + private static bool IsHidden(string path, string name) => + name.StartsWith('.') || (OperatingSystem.IsWindows() && File.GetAttributes(path).HasFlag(FileAttributes.Hidden)); +} diff --git a/src/Titano.Core/Archiving/ArchiveVolume.cs b/src/Titano.Core/Archiving/ArchiveVolume.cs new file mode 100644 index 00000000..d19a22a4 --- /dev/null +++ b/src/Titano.Core/Archiving/ArchiveVolume.cs @@ -0,0 +1,55 @@ +using Titano.Core.Localization; + +namespace Titano.Core.Archiving; + +/// The archive cannot be reached (or written) right now; nothing was changed. +public sealed class ArchiveOfflineException(string root, ArchiveAvailability availability) + : IOException(Loc.L(availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", root)) +{ + public string Root { get; } = root; + public ArchiveAvailability Availability { get; } = availability; +} + +public enum ArchiveAvailability +{ + Online, + + /// Readable but not writable (read-only share, write-protected disk): browse only. + ReadOnly, + + /// Root unreachable (unplugged drive, disconnected share) or no longer an archive. + Offline, +} + +/// Offline detection for archives on removable and network volumes (BRIEF §7). +public static class ArchiveVolume +{ + /// Throws unless the archive is online (writable). + public static void EnsureWritable(ArchivePaths archive) + { + var availability = Probe(archive); + if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(archive.Root, availability); + } + + /// Checks that archive.json is reachable and that the metadata folder accepts writes. + public static ArchiveAvailability Probe(ArchivePaths archive) + { + try + { + if (!File.Exists(archive.ManifestPath)) return ArchiveAvailability.Offline; + var probe = Path.Combine(archive.MetaDirectory, ".probe-" + Guid.NewGuid().ToString("N")[..8]); + using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1, FileOptions.DeleteOnClose)) + { + } + return ArchiveAvailability.Online; + } + catch (UnauthorizedAccessException) + { + return ArchiveAvailability.ReadOnly; + } + catch (IOException) + { + return File.Exists(archive.ManifestPath) ? ArchiveAvailability.ReadOnly : ArchiveAvailability.Offline; + } + } +} diff --git a/src/Titano.Core/Archiving/ContentHash.cs b/src/Titano.Core/Archiving/ContentHash.cs new file mode 100644 index 00000000..ea647a8c --- /dev/null +++ b/src/Titano.Core/Archiving/ContentHash.cs @@ -0,0 +1,44 @@ +using System.Security.Cryptography; + +namespace Titano.Core.Archiving; + +/// SHA-256 of files, streamed (originals can be hundreds of megabytes). +public static class ContentHash +{ + internal const int BufferSize = 1 << 20; + + /// Lower-case hex SHA-256 of the file. + /// Receives the number of bytes read since the previous call. + public static async Task Sha256Async(string path, Action? progress = null, CancellationToken cancellationToken = default) + { + await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, BufferSize, + FileOptions.Asynchronous | FileOptions.SequentialScan); + return await Sha256Async(stream, progress, cancellationToken).ConfigureAwait(false); + } + + public static async Task Sha256Async(Stream stream, Action? progress = null, CancellationToken cancellationToken = default) + { + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var buffer = new byte[BufferSize]; + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0) + { + hash.AppendData(buffer, 0, read); + progress?.Invoke(read); + } + return Convert.ToHexStringLower(hash.GetHashAndReset()); + } +} + +/// Size and modification time of a file, the cheap change detector of incremental scans (BRIEF §7). +public readonly record struct FileStamp(long Size, DateTime ModifiedUtc) +{ + /// The stamp of a file or directory, or null when it does not exist. + public static FileStamp? Of(string path) + { + var file = new FileInfo(path); + if (file.Exists) return new FileStamp(file.Length, file.LastWriteTimeUtc); + var directory = new DirectoryInfo(path); + return directory.Exists ? new FileStamp(0, directory.LastWriteTimeUtc) : null; + } +} diff --git a/src/Titano.Core/Archiving/FrameImageRenderer.cs b/src/Titano.Core/Archiving/FrameImageRenderer.cs new file mode 100644 index 00000000..0cdbfff3 --- /dev/null +++ b/src/Titano.Core/Archiving/FrameImageRenderer.cs @@ -0,0 +1,97 @@ +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; + +namespace Titano.Core.Archiving; + +/// A JPEG made from a frame. +public sealed record EncodedImage(byte[] Jpeg, int Width, int Height, DecodeTier Tier); + +/// +/// Makes thumbnails and proxies of frames (BRIEF §8.5): the embedded preview or the managed JPEG decoder first, then +/// the host's decode providers (WIC on Windows); orientation applied, resampled in linear light, encoded in-house. +/// +public sealed class FrameImageRenderer(ImageDecoder decoder) +{ + public const int ThumbnailLongEdge = 320; + public const int ThumbnailQuality = 80; + + /// A 320 px q80 thumbnail, or null when no decoder can read the frame. + public EncodedImage? Thumbnail(SequencePaths sequence, SidecarFrame frame, CancellationToken cancellationToken = default) + { + using var decoded = Decode(sequence.File(frame.File), frame, ThumbnailLongEdge, DecodePurpose.Thumbnail, preferPreview: true, cancellationToken); + if (decoded is null) return null; + using var small = Resampler.Thumbnail(decoded.Pixels, ThumbnailLongEdge); + return Encode(small, new JpegEncodeOptions { Quality = ThumbnailQuality }, decoded.Tier); + } + + /// A proxy at the policy's long edge and quality with capture time and camera in its Exif; null when undecodable. + public EncodedImage? Proxy(SequencePaths sequence, SidecarFrame frame, SidecarCamera? camera, ProxyPolicy policy, CancellationToken cancellationToken = default) + { + if (policy.LongEdge == ProxyPolicy.Disabled) throw new InvalidOperationException("Proxies are disabled for this archive."); + using var decoded = Decode(sequence.File(frame.File), frame, policy.LongEdge, DecodePurpose.Proxy, preferPreview: false, cancellationToken); + if (decoded is null) return null; + var source = decoded.Pixels; + var (w, h) = policy.LongEdge == ProxyPolicy.FullSize ? (source.Width, source.Height) : Resampler.FitWithin(source.Width, source.Height, policy.LongEdge); + using var resized = Resampler.Resize(source, w, h); + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata + { + Make = camera?.Make, + Model = camera?.Model, + BodySerial = camera?.Serial, + CapturedAt = frame.CapturedAt, + UtcOffsetMinutes = frame.UtcOffset, + ExposureTime = frame.Exposure, + FNumber = frame.FNumber, + Iso = frame.Iso, + FocalLength = frame.Focal, + ExposureBias = frame.Bias, + Software = "Titano", + }); + // The encoder takes the APP1 payload; ExifWriter returns the whole segment (marker and length first). + return Encode(resized, new JpegEncodeOptions { Quality = policy.Quality, App1Segments = [exif[4..]] }, decoded.Tier); + } + + private static EncodedImage Encode(ImageBuffer pixels, JpegEncodeOptions options, DecodeTier tier) => + new(JpegEncoder.Encode(pixels, options), pixels.Width, pixels.Height, tier); + + private DecodedFrame? Decode(string path, SidecarFrame frame, int maxLongEdge, DecodePurpose purpose, bool preferPreview, CancellationToken cancellationToken) + { + if (frame.Format == ImageFileFormat.Jpeg) return DecodeJpeg(File.ReadAllBytes(path), maxLongEdge, frame.Orientation, DecodeTier.Native); + if (preferPreview && ReadPreview(path, frame) is { } preview) return DecodeJpeg(preview, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview); + var request = new DecodeRequest(path, System.Math.Max(0, maxLongEdge), purpose, ApplyOrientation: true, AllowPreview: true); + if (decoder.Decode(request, cancellationToken) is { } decoded) return new DecodedFrame(decoded.Pixels, decoded.Tier); + return ReadPreview(path, frame) is { } fallback ? DecodeJpeg(fallback, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview) : null; + } + + private static DecodedFrame? DecodeJpeg(byte[] data, int maxLongEdge, int orientation, DecodeTier tier) + { + ImageBuffer decoded; + try + { + decoded = JpegDecoder.DecodeToLinear(data, new JpegDecodeOptions { MaxLongEdge = System.Math.Max(0, maxLongEdge) }, expandGray: true); + } + catch (InvalidDataException) + { + return null; + } + if (orientation is <= 1 or > 8) return new DecodedFrame(decoded, tier); + using (decoded) return new DecodedFrame(PixelConvert.ApplyOrientation(decoded, orientation), tier); + } + + private static byte[]? ReadPreview(string path, SidecarFrame frame) + { + if (frame.PreviewOffset is not { } offset || frame.PreviewLength is not ({ } length and > 0 and < int.MaxValue)) return null; + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 4096, FileOptions.RandomAccess); + if (offset < 0 || offset + length > stream.Length) return null; + var data = new byte[length]; + stream.Position = offset; + stream.ReadExactly(data); + return data; + } + + private sealed record DecodedFrame(ImageBuffer Pixels, DecodeTier Tier) : IDisposable + { + public void Dispose() => Pixels.Dispose(); + } +} diff --git a/src/Titano.Core/Archiving/FrameVerifier.cs b/src/Titano.Core/Archiving/FrameVerifier.cs new file mode 100644 index 00000000..553a8051 --- /dev/null +++ b/src/Titano.Core/Archiving/FrameVerifier.cs @@ -0,0 +1,54 @@ +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +public enum FrameHealth +{ + /// Same size and SHA-256 as recorded. + Ok, + + /// No hash was recorded (adopted file); it has been computed now. + Hashed, + + Missing, + + /// A different size: the file was replaced or edited. + Changed, + + /// Same size, different content: bit rot or a damaged copy. + Corrupt, +} + +/// The hash computed now (null when the file is missing or changed size). +public sealed record FrameCheck(SidecarFrame Frame, FrameHealth Health, string? Sha256) +{ + /// Frame flags after this check: / set or cleared. + public FrameFlags Flags => Health switch + { + FrameHealth.Missing => (Frame.Flags & ~FrameFlags.Corrupt) | FrameFlags.Missing, + FrameHealth.Changed or FrameHealth.Corrupt => (Frame.Flags & ~FrameFlags.Missing) | FrameFlags.Corrupt, + _ => Frame.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt), + }; + + /// The sidecar record after this check (flags, and the hash of a newly hashed file). + public SidecarFrame Updated => Frame with { Flags = Flags, Sha256 = Health == FrameHealth.Hashed ? Sha256 : Frame.Sha256 }; + + public bool ChangesRecord => Updated != Frame; +} + +/// Re-hashes originals against their sidecar records (BRIEF §12.5 Verify). +public static class FrameVerifier +{ + public static async Task CheckAsync(SequencePaths sequence, SidecarFrame frame, Action? bytesRead = null, + CancellationToken cancellationToken = default) + { + var info = new FileInfo(sequence.File(frame.File)); + if (!info.Exists) return new FrameCheck(frame, FrameHealth.Missing, null); + if (info.Length != frame.Size) return new FrameCheck(frame, FrameHealth.Changed, null); + var sha = await ContentHash.Sha256Async(info.FullName, bytesRead, cancellationToken).ConfigureAwait(false); + var health = frame.Sha256 is null ? FrameHealth.Hashed + : sha.Equals(frame.Sha256, StringComparison.OrdinalIgnoreCase) ? FrameHealth.Ok + : FrameHealth.Corrupt; + return new FrameCheck(frame, health, sha); + } +} diff --git a/src/Titano.Core/Archiving/IndexSnapshot.cs b/src/Titano.Core/Archiving/IndexSnapshot.cs new file mode 100644 index 00000000..55b54583 --- /dev/null +++ b/src/Titano.Core/Archiving/IndexSnapshot.cs @@ -0,0 +1,127 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using System.Text; +using Titano.Core.IO; + +namespace Titano.Core.Archiving; + +/// +/// .titano/index-snapshot.bin (BRIEF §7): every sidecar of the archive in one file, so a rebuild reads one file +/// sequentially instead of thousands of small ones over a share. Optional and never authoritative: an entry is used +/// only when its sidecar still has the recorded size and time, and a file whose trailing SHA-256 does not match is +/// ignored entirely. +/// +/// Layout: "TISN", version (u16), reserved (u16), then per sidecar: folder length (u16), UTF-8 folder, +/// size (i64), modified UTC ticks (i64), length (i32), sidecar bytes; finally the SHA-256 of everything before it. +public sealed class IndexSnapshot +{ + private const ushort Version = 1; + private static ReadOnlySpan Magic => "TISN"u8; + + private readonly Dictionary _entries; + + private IndexSnapshot(Dictionary entries) => _entries = entries; + + public int Count => _entries.Count; + + /// The snapshot at , or null when it is absent, damaged or of another version. + public static IndexSnapshot? TryLoad(string path) + { + byte[] data; + try + { + if (!File.Exists(path)) return null; + data = File.ReadAllBytes(path); + } + catch (IOException) + { + return null; + } + if (data.Length < 8 + 32 || !data.AsSpan(0, 4).SequenceEqual(Magic) || BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan(4)) != Version) + return null; + var body = data.AsSpan(0, data.Length - 32); + if (!SHA256.HashData(body).AsSpan().SequenceEqual(data.AsSpan(data.Length - 32))) return null; + + var entries = new Dictionary(StringComparer.Ordinal); + var position = 8; + while (position < body.Length) + { + if (position + 2 > body.Length) return null; + var folderLength = BinaryPrimitives.ReadUInt16LittleEndian(body[position..]); + position += 2; + if (position + folderLength + 20 > body.Length) return null; + var folder = Encoding.UTF8.GetString(body.Slice(position, folderLength)); + position += folderLength; + var size = BinaryPrimitives.ReadInt64LittleEndian(body[position..]); + var ticks = BinaryPrimitives.ReadInt64LittleEndian(body[(position + 8)..]); + var length = BinaryPrimitives.ReadInt32LittleEndian(body[(position + 16)..]); + position += 20; + if (length < 0 || ticks < 0 || ticks > DateTime.MaxValue.Ticks || position + length > body.Length) return null; + entries[folder] = (new FileStamp(size, new DateTime(ticks, DateTimeKind.Utc)), body.Slice(position, length).ToArray()); + position += length; + } + return new IndexSnapshot(entries); + } + + /// The recorded sidecar bytes of when its current stamp still matches. + public byte[]? TryGet(string folder, FileStamp current) => + _entries.TryGetValue(folder, out var entry) && entry.Stamp == current ? entry.Bytes : null; + + /// Streams a new snapshot to a temporary file; replaces the old one atomically. + public sealed class Writer : IDisposable + { + private readonly string _path; + private readonly string _temp; + private readonly FileStream _stream; + private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + private bool _committed; + + public Writer(string path) + { + _path = path; + _temp = AtomicFile.TempPathFor(path); + _stream = new FileStream(_temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16); + Span header = stackalloc byte[8]; + header.Clear(); + Magic.CopyTo(header); + BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version); + Write(header); + } + + public void Add(string folder, FileStamp stamp, ReadOnlySpan sidecar) + { + var name = Encoding.UTF8.GetBytes(folder); + Span head = stackalloc byte[2]; + BinaryPrimitives.WriteUInt16LittleEndian(head, checked((ushort)name.Length)); + Write(head); + Write(name); + Span fields = stackalloc byte[20]; + BinaryPrimitives.WriteInt64LittleEndian(fields, stamp.Size); + BinaryPrimitives.WriteInt64LittleEndian(fields[8..], stamp.ModifiedUtc.Ticks); + BinaryPrimitives.WriteInt32LittleEndian(fields[16..], sidecar.Length); + Write(fields); + Write(sidecar); + } + + public void Commit() + { + _stream.Write(_hash.GetHashAndReset()); + _stream.Dispose(); + AtomicFile.Commit(_temp, _path); + _committed = true; + } + + public void Dispose() + { + _stream.Dispose(); + _hash.Dispose(); + if (!_committed && File.Exists(_temp)) File.Delete(_temp); + } + + private void Write(ReadOnlySpan data) + { + _hash.AppendData(data); + _stream.Write(data); + } + } +} diff --git a/src/Titano.Core/Archiving/IngestJournal.cs b/src/Titano.Core/Archiving/IngestJournal.cs new file mode 100644 index 00000000..ec6baf25 --- /dev/null +++ b/src/Titano.Core/Archiving/IngestJournal.cs @@ -0,0 +1,52 @@ +using System.Text.Json; +using Titano.Core.IO; + +namespace Titano.Core.Archiving; + +/// +/// Write-ahead record of an ingest batch (.titano/ingest.pending.json): the frames whose files are about to be +/// moved into the sequence folder. Written before the moves and removed once sequence.json lists them, so a +/// crash in between is repaired by instead of leaving unindexed originals behind. +/// +public sealed record IngestJournal(IReadOnlyList Frames) +{ + public static void Write(SequencePaths sequence, IngestJournal journal) + { + Directory.CreateDirectory(sequence.MetaDirectory); + AtomicFile.WriteAllBytes(sequence.IngestJournalPath, JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJson.Default.IngestJournal)); + } + + public static void Delete(SequencePaths sequence) + { + if (File.Exists(sequence.IngestJournalPath)) File.Delete(sequence.IngestJournalPath); + } + + /// + /// Completes an interrupted batch: frames whose file reached the sequence folder (same size) but are missing from + /// are added in capture order. Returns the manifest to write, or null when nothing has + /// to change. The journal is left in place; it once the result is written. + /// + public static SequenceManifest? Recover(SequencePaths sequence, SequenceManifest manifest) + { + if (!File.Exists(sequence.IngestJournalPath)) return null; + IngestJournal? journal; + try + { + journal = JsonSerializer.Deserialize(File.ReadAllBytes(sequence.IngestJournalPath), ArchiveJson.Default.IngestJournal); + } + catch (JsonException) + { + journal = null; // torn while being written: nothing was moved yet + } + if (journal is null) return null; + var known = manifest.Frames.Select(f => f.Id).ToHashSet(); + var moved = journal.Frames.Where(f => !known.Contains(f.Id) && FileHasSize(sequence.File(f.File), f.Size)).ToList(); + return moved.Count == 0 ? null : SequenceFrames.Insert(manifest, moved); + } + + private static bool FileHasSize(string path, long size) + { + var info = new FileInfo(path); + return info.Exists && info.Length == size; + } +} diff --git a/src/Titano.Core/Archiving/LeaseFile.cs b/src/Titano.Core/Archiving/LeaseFile.cs new file mode 100644 index 00000000..7a460481 --- /dev/null +++ b/src/Titano.Core/Archiving/LeaseFile.cs @@ -0,0 +1,241 @@ +using System.Text.Json; +using Titano.Core.Diagnostics; +using Titano.Core.Localization; + +namespace Titano.Core.Archiving; + +/// Content of a lease file: who holds it and until when. +public sealed record LeaseRecord(string Token, string Owner, string Host, int ProcessId, DateTimeOffset Acquired, DateTimeOffset Expires); + +/// The lease is held by someone else and did not free up in time. +public sealed class LeaseUnavailableException(string name, LeaseRecord? holder) + : IOException(Loc.L("Archive.Error.Busy", holder?.Owner ?? "?")) +{ + public string Name { get; } = name; + public LeaseRecord? Holder { get; } = holder; +} + +/// +/// Cross-process exclusive lease stored as .titano/locks/<name>.lease (BRIEF §7), for writers that share +/// an archive over a network (server and client). Creation uses create-new semantics, which SMB and local file +/// systems both make atomic. A lease past its expiry is stale (its holder crashed or lost the share) and is taken +/// over by renaming it aside, which only one contender can do. The holder renews it in the background while held. +/// +public sealed class LeaseFile : IDisposable +{ + public static readonly TimeSpan DefaultDuration = TimeSpan.FromMinutes(2); + + private const string Extension = ".lease"; + private const string StaleExtension = ".stale"; + private readonly Lock _gate = new(); + private readonly Timer _renewal; + private LeaseRecord _record; + private bool _released; + + private LeaseFile(string path, string name, LeaseRecord record, TimeSpan duration) + { + Path = path; + Name = name; + _record = record; + Duration = duration; + var period = duration / 3; + _renewal = new Timer(_ => Renew(), null, period, period); + } + + public string Path { get; } + public string Name { get; } + public TimeSpan Duration { get; } + public LeaseRecord Record => _record; + + /// False once a renewal found the lease taken over (it expired while this process was stalled). + public bool IsHeld { get; private set; } = true; + + /// Takes the lease when it is free or stale; null when someone else holds it. + public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null) + { + var length = duration ?? DefaultDuration; + var path = PathFor(locksDirectory, name); + Directory.CreateDirectory(locksDirectory); + for (var attempt = 0; attempt < 3; attempt++) + { + var now = DateTimeOffset.UtcNow; + var record = new LeaseRecord(Guid.NewGuid().ToString("N"), owner, Environment.MachineName, Environment.ProcessId, now, now + length); + if (TryCreate(path, record)) return new LeaseFile(path, name, record, length); + if (!IsStale(path, now)) return null; + BreakStale(path, now); + } + return null; + } + + /// Waits up to for the lease, polling with back-off. + /// Still held by someone else after . + public static async Task AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait, + TimeSpan? duration = null, CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow + wait; + var delay = TimeSpan.FromMilliseconds(20); + while (true) + { + if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease; + if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name))); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, 500)); + } + } + + /// The current holder of a lease, or null when free or unreadable. + public static LeaseRecord? Read(string path) + { + try + { + return JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.LeaseRecord); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + return null; + } + } + + /// Removes every stale lease in (maintenance); returns their names. + public static IReadOnlyList CleanupStale(string locksDirectory, bool dryRun) + { + if (!Directory.Exists(locksDirectory)) return []; + var now = DateTimeOffset.UtcNow; + var removed = new List(); + foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + Extension)) + { + if (!IsStale(path, now)) continue; + removed.Add(System.IO.Path.GetFileNameWithoutExtension(path)); + if (!dryRun) BreakStale(path, now); + } + if (!dryRun) + { + foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + StaleExtension)) + { + if (File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime) TryDelete(path); + } + } + return removed; + } + + /// Extends the expiry; false when the lease is no longer ours. + public bool Renew() + { + lock (_gate) + { + if (_released || !IsHeld) return false; + if (Read(Path)?.Token != _record.Token) + { + IsHeld = false; + Log.Warning("archive", $"Lease {Name} was taken over by another writer"); + return false; + } + var renewed = _record with { Expires = DateTimeOffset.UtcNow + Duration }; + try + { + // In-place rewrite: a rename would open a window in which the lease looks free. + using var fs = new FileStream(Path, FileMode.Truncate, FileAccess.Write, FileShare.Read); + fs.Write(JsonSerializer.SerializeToUtf8Bytes(renewed, ArchiveJson.Default.LeaseRecord)); + fs.Flush(flushToDisk: true); + _record = renewed; + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Runs on a timer thread: an escaping exception would end the process. + Log.Warning("archive", $"Lease {Name} could not be renewed", ex); + return false; + } + } + } + + public void Dispose() + { + _renewal.Dispose(); + lock (_gate) + { + if (_released) return; + _released = true; + if (!IsHeld || Read(Path)?.Token != _record.Token) return; + try + { + File.Delete(Path); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // It expires on its own. + Log.Warning("archive", $"Lease {Name} could not be released", ex); + } + } + } + + internal static string PathFor(string locksDirectory, string name) => System.IO.Path.Combine(locksDirectory, name + Extension); + + private static bool TryCreate(string path, LeaseRecord record) + { + try + { + using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read); + fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord)); + fs.Flush(flushToDisk: true); + return true; + } + catch (IOException) when (File.Exists(path)) + { + return false; + } + } + + /// Expired, or unreadable for longer than a lease lasts (a holder that crashed while creating it). + private static bool IsStale(string path, DateTimeOffset now) + { + if (Read(path) is { } record) return record.Expires < now; + try + { + return File.Exists(path) && File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime; + } + catch (IOException) + { + return false; + } + } + + private static void BreakStale(string path, DateTimeOffset now) + { + var aside = path + "." + Guid.NewGuid().ToString("N")[..8] + StaleExtension; + try + { + File.Move(path, aside); + } + catch (IOException) + { + return; // another contender broke it first + } + if (IsStale(aside, now)) + { + TryDelete(aside); + return; + } + // A contender renewed or re-created the lease between our check and the rename: give it back. + try + { + File.Move(aside, path, overwrite: false); + } + catch (IOException) + { + TryDelete(aside); + } + } + + private static void TryDelete(string path) + { + try + { + File.Delete(path); + } + catch (IOException) + { + // A leftover file is harmless; maintenance removes it. + } + } +} diff --git a/src/Titano.Core/Archiving/ManifestStore.cs b/src/Titano.Core/Archiving/ManifestStore.cs new file mode 100644 index 00000000..62a3a11b --- /dev/null +++ b/src/Titano.Core/Archiving/ManifestStore.cs @@ -0,0 +1,84 @@ +using System.Text.Json; +using Titano.Core.IO; +using Titano.Core.Localization; + +namespace Titano.Core.Archiving; + +/// A sidecar written by a newer Titano: it can be read, but this build refuses to rewrite it. +public sealed class NewerFormatException(string path, int version) + : InvalidOperationException(Loc.L("Archive.Error.NewerFormat", System.IO.Path.GetFileName(path), version)) +{ + public string Path { get; } = path; + public int Version { get; } = version; +} + +/// +/// Reads and writes the JSON files of the layout. Reading tolerates unknown fields and enum values; every write +/// goes through (temp → fsync → rename), so a crash leaves the previous file intact. +/// +public static class ManifestStore +{ + /// The file is not valid JSON or lacks required fields. + public static ArchiveManifest ReadArchive(string path) => Parse(File.ReadAllBytes(path), path, ArchiveJson.Default.ArchiveManifest); + + public static void WriteArchive(string path, ArchiveManifest manifest) + { + RefuseNewer(path, manifest.SchemaVersion, ArchiveManifest.CurrentSchemaVersion); + AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJsonIndented.Default.ArchiveManifest)); + } + + /// The file is not valid JSON or lacks required fields. + public static SequenceManifest ReadSequence(string path) => ParseSequence(File.ReadAllBytes(path), path); + + /// Parses sidecar bytes (from disk or from the index snapshot). + public static SequenceManifest ParseSequence(ReadOnlySpan utf8, string source) => Parse(utf8, source, ArchiveJson.Default.SequenceManifest); + + /// Replaces the sidecar atomically. Compact JSON: a 100k-frame sidecar stays around 35 MB. + public static void WriteSequence(string path, SequenceManifest manifest) + { + RefuseNewer(path, manifest.SchemaVersion, SequenceManifest.CurrentSchemaVersion); + AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest)); + } + + /// + /// Creates the sidecar only if none exists (rename without replace), which makes it the claim on a new + /// sequence folder between processes sharing the archive. False when another sidecar is already there. + /// + public static bool TryCreateSequence(string path, SequenceManifest manifest) + { + if (File.Exists(path)) return false; + var temp = AtomicFile.TempPathFor(path); + File.WriteAllBytes(temp, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest)); + try + { + using (var fs = new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) fs.Flush(flushToDisk: true); + File.Move(temp, path, overwrite: false); + return true; + } + catch (IOException) when (File.Exists(path)) + { + return false; + } + finally + { + if (File.Exists(temp)) File.Delete(temp); + } + } + + private static T Parse(ReadOnlySpan utf8, string source, System.Text.Json.Serialization.Metadata.JsonTypeInfo typeInfo) + { + try + { + return JsonSerializer.Deserialize(utf8, typeInfo) ?? throw new InvalidDataException($"{source} is empty."); + } + catch (JsonException ex) + { + throw new InvalidDataException($"{source} is not a valid Titano file: {ex.Message}", ex); + } + } + + private static void RefuseNewer(string path, int version, int supported) + { + if (version > supported) throw new NewerFormatException(path, version); + } +} diff --git a/src/Titano.Core/Archiving/MassRename.cs b/src/Titano.Core/Archiving/MassRename.cs new file mode 100644 index 00000000..caf423c9 --- /dev/null +++ b/src/Titano.Core/Archiving/MassRename.cs @@ -0,0 +1,131 @@ +using System.Globalization; +using System.Text.Json; +using Titano.Core.IO; +using Titano.Core.Naming; + +namespace Titano.Core.Archiving; + +/// What the planner needs to know about a sequence to rename it. +public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens); + +/// One sequence's change: its folder, its title, or both. +public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle) +{ + public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal); +} + +/// The exact changes a mass rename will make (its dry run); applying it makes exactly these. +public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList Steps); + +/// A plan being applied, with the steps already completed; lets an interrupted rename resume (BRIEF §7). +public sealed record RenameJournal(RenamePlan Plan, IReadOnlyList Completed) +{ + public static string PathFor(ArchivePaths archive, Guid planId) => Path.Combine(archive.JournalDirectory, $"rename-{planId:N}.json"); + + public static void Write(ArchivePaths archive, RenameJournal journal) + { + Directory.CreateDirectory(archive.JournalDirectory); + AtomicFile.WriteAllBytes(PathFor(archive, journal.Plan.Id), JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJsonIndented.Default.RenameJournal)); + } + + /// Journals left by interrupted renames. + public static IReadOnlyList ListPending(ArchivePaths archive) + { + if (!Directory.Exists(archive.JournalDirectory)) return []; + var list = new List(); + foreach (var path in Directory.EnumerateFiles(archive.JournalDirectory, "rename-*.json").Order(StringComparer.Ordinal)) + { + if (JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.RenameJournal) is { } journal) list.Add(journal); + } + return list; + } + + public static void Delete(ArchivePaths archive, Guid planId) => File.Delete(PathFor(archive, planId)); +} + +/// Builds rename plans from folder/title templates (BRIEF §7.1, §12.5). +public static class RenamePlanner +{ + /// + /// Expands (where {title} is the current title) and then + /// (where {title} is the new title) for every subject. A target folder + /// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"…. + /// Null templates leave that part unchanged; subjects without a change produce no step. + /// + /// Every sequence folder of the archive (renamed or not). + public static RenamePlan Plan(Guid archiveId, IReadOnlyList subjects, string? folderTemplate, string? titleTemplate, + IEnumerable occupied, Func existsOnDisk) + { + var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate); + var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate); + var claimed = new HashSet(occupied, StringComparer.OrdinalIgnoreCase); + var steps = new List(); + foreach (var s in subjects) + { + var context = new NamingContext + { + CapturedAt = s.FirstCapture ?? DateTime.Today, + SessionStart = s.FirstCapture ?? DateTime.Today, + CameraMake = s.CameraMake, + CameraModel = s.CameraModel, + Lens = s.Lens, + Title = s.Title, + }; + var newTitle = title is null ? s.Title : Blank(title.ExpandFile(context)) ?? s.Title; + var newFolder = s.Folder; + // The archive root itself (a folder adopted in place) cannot move; only its title changes. + if (folder is not null && s.Folder.Length > 0 && Blank(folder.ExpandFolder(context with { Title = newTitle })) is { } expanded + && !expanded.Equals(s.Folder, StringComparison.Ordinal)) + { + newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk); + claimed.Add(newFolder); + } + if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle)); + } + return new RenamePlan(Guid.CreateVersion7(), archiveId, steps); + } + + /// + /// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted, + /// a case-only change goes through a temporary name (case-insensitive file systems). + /// + /// The target is occupied by something else, or neither folder exists. + public static void MoveFolder(ArchivePaths archive, RenameStep step) + { + if (!step.MovesFolder) return; + var from = archive.FromRelative(step.FromFolder); + var to = archive.FromRelative(step.ToFolder); + var caseOnly = step.FromFolder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase); + if (Directory.Exists(to) && (!caseOnly || HasExactName(to))) + { + if (Directory.Exists(from) && !caseOnly) throw new IOException($"Rename target {step.ToFolder} already exists."); + return; + } + if (!Directory.Exists(from)) throw new DirectoryNotFoundException($"Sequence folder {step.FromFolder} is missing."); + Directory.CreateDirectory(Path.GetDirectoryName(to)!); + if (caseOnly) + { + var temp = from + ".rename-" + Guid.NewGuid().ToString("N")[..8]; + Directory.Move(from, temp); + Directory.Move(temp, to); + } + else + { + Directory.Move(from, to); + } + } + + private static bool HasExactName(string path) => + Directory.EnumerateDirectories(Path.GetDirectoryName(path)!, Path.GetFileName(path)) + .Any(d => Path.GetFileName(d).Equals(Path.GetFileName(path), StringComparison.Ordinal)); + + private static string? Blank(string value) => string.IsNullOrWhiteSpace(value) ? null : value; + + private static string Unique(string folder, HashSet claimed, Func existsOnDisk) + { + var candidate = folder; + for (var n = 2; claimed.Contains(candidate) || existsOnDisk(candidate); n++) + candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})"); + return candidate; + } +} diff --git a/src/Titano.Core/Archiving/SequenceAdopter.cs b/src/Titano.Core/Archiving/SequenceAdopter.cs new file mode 100644 index 00000000..77d38d90 --- /dev/null +++ b/src/Titano.Core/Archiving/SequenceAdopter.cs @@ -0,0 +1,91 @@ +using Titano.Core.Diagnostics; +using Titano.Core.Metadata; +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +/// Files of a sequence folder that the sidecar does not list, and sidecar frames whose file is gone. +public sealed record SequenceOrphans(string Folder, IReadOnlyList UnindexedFiles, IReadOnlyList MissingFiles) +{ + public bool IsEmpty => UnindexedFiles.Count == 0 && MissingFiles.Count == 0; +} + +/// +/// Builds and reconciles sidecars from the photos found in a folder (BRIEF §7, §12.5 "Adopt folder"). Only light +/// metadata is read; originals are never moved, renamed or rewritten. Hashes are left empty for a verify to fill. +/// +public static class SequenceAdopter +{ + /// A new sidecar for a folder without one. A folder outside the layout (no originals/) is titled by its name. + public static SequenceManifest Build(Guid id, SequencePaths sequence, string? createdBy, CancellationToken cancellationToken = default) + { + var (frames, first) = ReadFrames(sequence, ArchiveScanner.ListPhotos(sequence), cancellationToken); + var now = DateTimeOffset.UtcNow; + var inLayout = Directory.Exists(sequence.Originals); + var manifest = new SequenceManifest + { + Id = id, + Title = inLayout ? null : Path.GetFileName(Path.TrimEndingDirectorySeparator(sequence.Folder)), + Camera = first is null ? null : new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, + Lens = first?.LensModel, + CreatedBy = createdBy, + Created = now, + UpdatedAt = now, + }; + return SequenceFrames.Insert(manifest, frames); + } + + /// Unindexed photos and missing frame files of one sequence. + public static SequenceOrphans FindOrphans(SequencePaths sequence, SequenceManifest manifest) + { + var listed = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal); + var unindexed = ArchiveScanner.ListPhotos(sequence).Where(f => !listed.Contains(f)).ToList(); + var missing = manifest.Frames.Where(f => !File.Exists(sequence.File(f.File))).ToList(); + return new SequenceOrphans(sequence.RelativeFolder, unindexed, missing); + } + + /// + /// Brings a sidecar in line with its folder: unindexed photos are added (in capture order), frames whose file is + /// gone get and frames whose file came back lose it. Null when nothing changed. + /// + public static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken = default) + { + var orphans = FindOrphans(sequence, manifest); + var missing = orphans.MissingFiles.Select(f => f.Id).ToHashSet(); + var flagChanges = new Dictionary>(); + foreach (var frame in manifest.Frames) + { + var isMissing = missing.Contains(frame.Id); + if (isMissing == frame.Flags.HasFlag(FrameFlags.Missing)) continue; + flagChanges[frame.Id] = f => f with { Flags = isMissing ? f.Flags | FrameFlags.Missing : f.Flags & ~FrameFlags.Missing }; + } + if (flagChanges.Count == 0 && orphans.UnindexedFiles.Count == 0) return null; + var updated = flagChanges.Count == 0 ? manifest : SequenceFrames.Update(manifest, flagChanges); + var (added, first) = ReadFrames(sequence, orphans.UnindexedFiles, cancellationToken); + if (updated.Camera is null && first is not null) + updated = updated with { Camera = new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, Lens = updated.Lens ?? first.LensModel }; + return SequenceFrames.Insert(updated, added); + } + + private static (List Frames, FrameMetadata? First) ReadFrames(SequencePaths sequence, IReadOnlyList files, CancellationToken cancellationToken) + { + var frames = new List(files.Count); + FrameMetadata? first = null; + foreach (var file in files) + { + cancellationToken.ThrowIfCancellationRequested(); + var path = sequence.File(file); + try + { + var metadata = MetadataReader.Read(path, cancellationToken); + if (first is null || (metadata.CapturedAt < first.CapturedAt && metadata.Make is not null)) first = metadata; + frames.Add(SequenceFrames.FromMetadata(Guid.CreateVersion7(), file, Path.GetFileName(file), new FileInfo(path).Length, null, metadata)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.Warning("archive", $"Skipped unreadable photo {sequence.ArchiveRelative(file)}", ex); + } + } + return (frames, first); + } +} diff --git a/src/Titano.Core/Archiving/SequenceFrames.cs b/src/Titano.Core/Archiving/SequenceFrames.cs new file mode 100644 index 00000000..4735e3ae --- /dev/null +++ b/src/Titano.Core/Archiving/SequenceFrames.cs @@ -0,0 +1,86 @@ +using Titano.Core.Metadata; +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +/// Edits of the frame list of a ; frames stay in capture order with gap-free indexes. +public static class SequenceFrames +{ + /// Capture time (unknown last), then file name. + public static readonly IComparer CaptureOrder = Comparer.Create((a, b) => + { + var byTime = (a.CapturedAt ?? DateTime.MaxValue).CompareTo(b.CapturedAt ?? DateTime.MaxValue); + return byTime != 0 ? byTime : string.CompareOrdinal(a.File, b.File); + }); + + /// A frame record from light metadata. + /// Path relative to the sequence folder. + /// Used when the file records no capture time of its own (only a file-system date). + public static SidecarFrame FromMetadata(Guid id, string file, string? original, long size, string? sha256, FrameMetadata metadata, + DateTime? fallbackCapture = null) + { + var useFallback = fallbackCapture is not null && metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown; + return new SidecarFrame + { + Id = id, + File = file, + Original = original, + Size = size, + Sha256 = sha256?.ToLowerInvariant(), + CapturedAt = useFallback ? DateTime.SpecifyKind(fallbackCapture!.Value, DateTimeKind.Unspecified) : metadata.CapturedAt, + UtcOffset = useFallback ? null : metadata.UtcOffsetMinutes, + TimeSource = useFallback ? TimestampSource.Unknown : metadata.TimestampSource, + Exposure = metadata.ExposureTime, + FNumber = metadata.FNumber, + Iso = metadata.Iso, + Focal = metadata.FocalLength, + Bias = metadata.ExposureBias, + Wb = metadata.WhiteBalanceMode, + Orientation = metadata.Orientation, + W = metadata.Width, + H = metadata.Height, + Format = metadata.Format, + PreviewOffset = metadata.LargestPreview?.Offset, + PreviewLength = metadata.LargestPreview?.Length, + }; + } + + /// + /// Merges into the frame list by capture time (existing frames keep their relative order), + /// renumbers indexes and bumps the frames version. + /// + public static SequenceManifest Insert(SequenceManifest manifest, IReadOnlyList added) + { + if (added.Count == 0) return manifest; + var incoming = added.Order(CaptureOrder).ToList(); + var merged = new List(manifest.Frames.Count + incoming.Count); + var next = 0; + foreach (var frame in manifest.Frames) + { + while (next < incoming.Count && CaptureOrder.Compare(incoming[next], frame) < 0) merged.Add(incoming[next++]); + merged.Add(frame); + } + while (next < incoming.Count) merged.Add(incoming[next++]); + return WithFrames(manifest, merged); + } + + /// Replaces the frame list (indexes renumbered) and bumps the frames version. + public static SequenceManifest WithFrames(SequenceManifest manifest, IReadOnlyList frames) + { + var renumbered = new SidecarFrame[frames.Count]; + for (var i = 0; i < frames.Count; i++) renumbered[i] = frames[i].I == i ? frames[i] : frames[i] with { I = i }; + return manifest with + { + Frames = renumbered, + Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 }, + UpdatedAt = DateTimeOffset.UtcNow, + }; + } + + /// Applies per-frame changes by id (flags, hashes); frames not in are kept. + public static SequenceManifest Update(SequenceManifest manifest, IReadOnlyDictionary> changes) + { + var frames = manifest.Frames.Select(f => changes.TryGetValue(f.Id, out var change) ? change(f) : f).ToList(); + return manifest with { Frames = frames, UpdatedAt = DateTimeOffset.UtcNow }; + } +} diff --git a/src/Titano.Core/Archiving/SequenceManifest.cs b/src/Titano.Core/Archiving/SequenceManifest.cs new file mode 100644 index 00000000..dbab9390 --- /dev/null +++ b/src/Titano.Core/Archiving/SequenceManifest.cs @@ -0,0 +1,117 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using Titano.Core.Imaging; +using Titano.Core.Model; + +namespace Titano.Core.Archiving; + +/// +/// .titano/sequence.json, the canonical record of one sequence (BRIEF §7). The database is rebuilt from it, +/// so it carries the ids of the sequence and of every frame. Fields this build does not know are kept in +/// and written back unchanged. +/// +public sealed record SequenceManifest +{ + public const int CurrentSchemaVersion = 1; + + public int SchemaVersion { get; init; } = CurrentSchemaVersion; + public required Guid Id { get; init; } + public string? Title { get; init; } + public SidecarCamera? Camera { get; init; } + public string? Lens { get; init; } + + /// Frames in capture order; equals the position. + public IReadOnlyList Frames { get; init; } = []; + + public IReadOnlyList Tags { get; init; } = []; + public int Rating { get; init; } + public string? Notes { get; init; } + + /// bits, stored as a number so bits of newer builds survive. + public SequenceStatus Status { get; init; } + + public ManifestVersions Versions { get; init; } = new(); + + /// User id or client name that created the sequence. + public string? CreatedBy { get; init; } + + public DateTimeOffset Created { get; init; } + public DateTimeOffset UpdatedAt { get; init; } + public DateTimeOffset? VerifiedAt { get; init; } + + /// Session keys of automatic grouping (uploads, card imports) that resolve to this sequence. + public IReadOnlyList SessionKeys { get; init; } = []; + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +public sealed record SidecarCamera +{ + public string? Make { get; init; } + public string? Model { get; init; } + public string? Serial { get; init; } + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// Counters the sync protocol compares (BRIEF §14.5). +public sealed record ManifestVersions +{ + public int Settings { get; init; } + public int Frames { get; init; } + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} + +/// One frame of ; property names follow BRIEF §7. +public sealed record SidecarFrame +{ + public required Guid Id { get; init; } + + /// Index in capture order, from 0. + public int I { get; init; } + + /// Path relative to the sequence folder, forward slashes (originals/… for imported frames). + public required string File { get; init; } + + /// Name the file had on the card or upload source. + public string? Original { get; init; } + + public long Size { get; init; } + + /// Lower-case hex SHA-256; null until a verify hashes an adopted file. + public string? Sha256 { get; init; } + + /// Camera wall-clock time with sub-seconds. + public DateTime? CapturedAt { get; init; } + + public int? UtcOffset { get; init; } + + [JsonConverter(typeof(TolerantEnumConverter))] + public TimestampSource TimeSource { get; init; } + + public double? Exposure { get; init; } + public double? FNumber { get; init; } + public int? Iso { get; init; } + public double? Focal { get; init; } + public double? Bias { get; init; } + public int? Wb { get; init; } + public int Orientation { get; init; } = 1; + public int W { get; init; } + public int H { get; init; } + + [JsonConverter(typeof(TolerantEnumConverter))] + public ImageFileFormat Format { get; init; } + + public long? PreviewOffset { get; init; } + public long? PreviewLength { get; init; } + + /// bits, stored as a number so bits of newer builds survive. + public FrameFlags Flags { get; init; } + + [JsonExtensionData] + public Dictionary? Extra { get; set; } +} diff --git a/src/Titano.Core/Archiving/ThumbPack.cs b/src/Titano.Core/Archiving/ThumbPack.cs new file mode 100644 index 00000000..48bf29df --- /dev/null +++ b/src/Titano.Core/Archiving/ThumbPack.cs @@ -0,0 +1,389 @@ +using System.Buffers.Binary; +using Titano.Core.Hashing; + +namespace Titano.Core.Archiving; + +/// One thumbnail of a pack: where its JPEG lies and which source it was made from. +/// 32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale. +public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc); + +/// +/// thumbs.pack: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence +/// instead of thousands. Little-endian blocks after a 16-byte header: +/// +/// TENT entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes. +/// TIDX index: count, one row per frame (latest entry wins), CRC-32 of the rows. +/// TFTR footer (last 16 bytes): offset of the index block and its CRC. +/// +/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index +/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall +/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it). +/// +public static class ThumbPack +{ + public const int HashLength = 32; + internal const int HeaderLength = 16; + internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4; + internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4; + internal const int FooterLength = 16; + internal const ushort Version = 1; + + /// Largest thumbnail accepted; anything bigger in a header is corruption. + internal const int MaxEntryLength = 16 << 20; + + internal static ReadOnlySpan Magic => "TTPK"u8; + internal static ReadOnlySpan EntryTag => "TENT"u8; + internal static ReadOnlySpan IndexTag => "TIDX"u8; + internal static ReadOnlySpan FooterTag => "TFTR"u8; + + internal static void WriteHeader(Stream stream) + { + Span header = stackalloc byte[HeaderLength]; + header.Clear(); + Magic.CopyTo(header); + BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version); + stream.Write(header); + } + + internal static bool HasValidHeader(Stream stream) + { + if (stream.Length < HeaderLength) return false; + Span header = stackalloc byte[HeaderLength]; + stream.Position = 0; + stream.ReadExactly(header); + return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version; + } + + /// + /// The latest entry per frame and where the valid content ends: after the footer when the index is intact + /// ( = start of that index), otherwise after the last complete block. + /// + internal static Dictionary Load(Stream stream, out long validEnd, out long? indexOffset) + { + indexOffset = null; + if (TryLoadIndex(stream, out var entries, out var offset)) + { + validEnd = stream.Length; + indexOffset = offset; + return entries; + } + return Scan(stream, out validEnd); + } + + private static bool TryLoadIndex(Stream stream, out Dictionary entries, out long indexOffset) + { + entries = []; + indexOffset = 0; + var length = stream.Length; + if (length < HeaderLength + 12 + FooterLength) return false; + Span footer = stackalloc byte[FooterLength]; + stream.Position = length - FooterLength; + stream.ReadExactly(footer); + if (!footer[..4].SequenceEqual(FooterTag)) return false; + indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]); + if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false; + if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false; + + Span head = stackalloc byte[8]; + stream.Position = indexOffset; + stream.ReadExactly(head); + var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]); + if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false; + var rows = new byte[count * IndexRowLength + 4]; + stream.ReadExactly(rows); + if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false; + for (var i = 0; i < count; i++) + { + var row = rows.AsSpan(i * IndexRowLength, IndexRowLength); + var entry = new ThumbEntry( + BinaryPrimitives.ReadInt32LittleEndian(row), + BinaryPrimitives.ReadInt64LittleEndian(row[4..]), + BinaryPrimitives.ReadInt32LittleEndian(row[12..]), + BinaryPrimitives.ReadUInt16LittleEndian(row[16..]), + BinaryPrimitives.ReadUInt16LittleEndian(row[18..]), + row.Slice(20, HashLength).ToArray(), + BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..])); + if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false; + entries[entry.FrameIndex] = entry; + } + return true; + } + + /// Walks block headers from the start; stops at the first incomplete or unknown block. + private static Dictionary Scan(Stream stream, out long validEnd) + { + var entries = new Dictionary(); + var length = stream.Length; + var position = (long)HeaderLength; + var header = new byte[EntryHeaderLength]; + while (position + 4 <= length) + { + stream.Position = position; + stream.ReadExactly(header.AsSpan(0, 4)); + var tag = header.AsSpan(0, 4); + long next; + if (tag.SequenceEqual(EntryTag)) + { + if (position + EntryHeaderLength > length) break; + stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4)); + var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength)); + if (size is < 0 or > MaxEntryLength) break; + next = position + EntryHeaderLength + size; + if (next > length) break; + var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4)); + entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size, + BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)), + header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength))); + } + else if (tag.SequenceEqual(IndexTag)) + { + if (position + 8 > length) break; + stream.ReadExactly(header.AsSpan(4, 4)); + var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4)); + if (count < 0) break; + next = position + 8 + (long)count * IndexRowLength + 4; + if (next > length) break; + } + else if (tag.SequenceEqual(FooterTag)) + { + next = position + FooterLength; + if (next > length) break; + } + else + { + break; + } + position = next; + } + validEnd = position; + return entries; + } + + /// Writes one entry block at the stream position and returns its entry. + internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan jpeg) + { + var crc = Crc32.Compute(jpeg); + Span header = stackalloc byte[EntryHeaderLength]; + EntryTag.CopyTo(header); + BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex); + BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width); + BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height); + sourceHash.CopyTo(header[12..]); + BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length); + BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc); + var payload = stream.Position + EntryHeaderLength; + stream.Write(header); + stream.Write(jpeg); + return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc); + } + + internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection entries) + { + var indexOffset = stream.Position; + var block = new byte[8 + entries.Count * IndexRowLength + 4]; + IndexTag.CopyTo(block); + BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count); + var i = 0; + foreach (var e in entries.OrderBy(e => e.FrameIndex)) + { + var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength); + BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex); + BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset); + BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length); + BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width); + BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height); + e.SourceHash.CopyTo(row[20..]); + BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc); + } + var rowsLength = entries.Count * IndexRowLength; + BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength))); + stream.Write(block); + + Span footer = stackalloc byte[FooterLength]; + FooterTag.CopyTo(footer); + BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset); + BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12])); + stream.Write(footer); + } + + /// Size of a pack holding only ; the rest of a file is reclaimable garbage. + public static long LiveBytes(IEnumerable entries) + { + long count = 0, payload = 0; + foreach (var e in entries) + { + count++; + payload += EntryHeaderLength + e.Length; + } + return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength; + } + + /// A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known. + public static byte[] Fingerprint(string? sha256, string file, long size) => + sha256 is { Length: HashLength * 2 } + ? Convert.FromHexString(sha256) + : System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}")); +} + +/// Random access to the thumbnails of a pack without loading the file; safe while a writer appends. +public sealed class ThumbPackReader : IDisposable +{ + private readonly FileStream? _stream; + private readonly Lock _gate = new(); + + private ThumbPackReader(FileStream? stream, Dictionary entries) + { + _stream = stream; + Entries = entries; + } + + /// The latest entry of every frame that has one. + public IReadOnlyDictionary Entries { get; } + + /// Bytes held by replaced entries (reclaimed by ). + public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values)); + + public long Length => _stream?.Length ?? 0; + + /// Opens a pack; a missing or damaged pack reads as empty. + public static ThumbPackReader Open(string path) + { + if (!File.Exists(path)) return new ThumbPackReader(null, []); + var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess); + try + { + var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : []; + return new ThumbPackReader(stream, entries); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// The JPEG of a frame, or null when absent or damaged (CRC mismatch). + public byte[]? Read(int frameIndex) + { + if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null; + var data = new byte[entry.Length]; + lock (_gate) + { + if (entry.Offset + entry.Length > _stream.Length) return null; + _stream.Position = entry.Offset; + _stream.ReadExactly(data); + } + return Crc32.Compute(data) == entry.Crc ? data : null; + } + + public void Dispose() => _stream?.Dispose(); +} + +/// Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it). +public sealed class ThumbPackWriter : IDisposable +{ + private readonly FileStream _stream; + private readonly Dictionary _entries; + private long? _indexOffset; + private bool _dirty; + + private ThumbPackWriter(FileStream stream, Dictionary entries, long? indexOffset) + { + _stream = stream; + _entries = entries; + _indexOffset = indexOffset; + } + + public IReadOnlyDictionary Entries => _entries; + + /// Opens or creates a pack, dropping an incomplete tail left by a crash. + public static ThumbPackWriter Open(string path) + { + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!); + var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16); + try + { + if (!ThumbPack.HasValidHeader(stream)) + { + // Not a pack (or a header torn by a crash): thumbnails are regenerable, start over. + stream.SetLength(0); + ThumbPack.WriteHeader(stream); + return new ThumbPackWriter(stream, [], null); + } + var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset); + if (validEnd < stream.Length) stream.SetLength(validEnd); + return new ThumbPackWriter(stream, entries, indexOffset); + } + catch + { + stream.Dispose(); + throw; + } + } + + /// Appends a thumbnail; it replaces any earlier entry of the frame once committed. + public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan jpeg) + { + ArgumentOutOfRangeException.ThrowIfNegative(frameIndex); + if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash)); + if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg)); + if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width)); + // The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk). + _stream.Position = _indexOffset ?? _stream.Length; + if (_indexOffset is { } offset) _stream.SetLength(offset); + _indexOffset = null; + + _entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg); + _dirty = true; + } + + /// Writes the index and footer and flushes to disk. + public void Commit() + { + if (!_dirty) return; + _stream.Position = _stream.Length; + var indexOffset = _stream.Position; + ThumbPack.WriteIndexAndFooter(_stream, _entries.Values); + _stream.Flush(flushToDisk: true); + _indexOffset = indexOffset; + _dirty = false; + } + + /// Rewrites the pack with only its live entries (atomically, so readers keep a consistent file). + public static void Compact(string path) + { + var temp = IO.AtomicFile.TempPathFor(path); + try + { + using (var source = ThumbPackReader.Open(path)) + using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16)) + { + ThumbPack.WriteHeader(target); + var entries = new List(source.Entries.Count); + foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex)) + { + if (source.Read(entry.FrameIndex) is { } jpeg) + entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg)); + } + ThumbPack.WriteIndexAndFooter(target, entries); + } + IO.AtomicFile.Commit(temp, path); + } + finally + { + if (File.Exists(temp)) File.Delete(temp); + } + } + + public void Dispose() + { + try + { + Commit(); + } + finally + { + _stream.Dispose(); + } + } +} diff --git a/tests/Titano.Core.Tests/Archiving/ArchiveFolderTests.cs b/tests/Titano.Core.Tests/Archiving/ArchiveFolderTests.cs new file mode 100644 index 00000000..78748418 --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/ArchiveFolderTests.cs @@ -0,0 +1,182 @@ +using Titano.Core.Archiving; +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; +using Titano.Core.Model; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class ArchiveFolderTests(TestContext context) +{ + private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0); + + private ArchivePaths NewArchive() => new(context.CreateTempDirectory("archive")); + + [Test] + public void SequenceFoldersAreFoundInLayoutAndAdoptedForms() + { + var archive = NewArchive(); + TestPhotos.WriteSeries(Path.Combine(archive.Root, "2026", "2026-09-28 Canon EOS R5", "originals"), Start, 3, TimeSpan.FromSeconds(10)); + TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour"), Start, 2, TimeSpan.FromSeconds(10)); + TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour", "selects"), Start, 1, TimeSpan.FromSeconds(10)); + TestPhotos.WriteSeries(Path.Combine(archive.Root, ".hidden"), Start, 1, TimeSpan.FromSeconds(10)); + File.WriteAllBytes(Path.Combine(archive.Root, "Trips", "._IMG_0001.JPG"), [1]); + Directory.CreateDirectory(Path.Combine(archive.Root, "empty")); + + var folders = ArchiveScanner.FindSequenceFolders(archive); + Assert.SequenceEqual(["2026/2026-09-28 Canon EOS R5", "Trips/Harbour", "Trips/Harbour/selects"], folders.Select(f => f.RelativeFolder)); + Assert.True(folders.All(f => f.Sidecar is null && f.Photos is not null)); + Assert.SequenceEqual(["originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG"], + ArchiveScanner.ListPhotos(archive.Sequence("2026/2026-09-28 Canon EOS R5"))); + } + + [Test] + public void AdoptionBuildsASidecarInCaptureOrderWithoutTouchingPhotos() + { + var archive = NewArchive(); + var folder = Path.Combine(archive.Root, "Harbour sunset"); + // File names run against capture order: the sidecar follows capture time. + TestPhotos.Write(Path.Combine(folder, "a.jpg"), Start.AddSeconds(20), 1); + TestPhotos.Write(Path.Combine(folder, "b.jpg"), Start, 2); + TestPhotos.Write(Path.Combine(folder, "c.jpg"), Start.AddSeconds(10), 3); + var before = Directory.GetFiles(folder).ToDictionary(f => f, File.ReadAllBytes); + + var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), archive.Sequence("Harbour sunset"), "test"); + Assert.SequenceEqual(["b.jpg", "c.jpg", "a.jpg"], manifest.Frames.Select(f => f.File)); + Assert.SequenceEqual([0, 1, 2], manifest.Frames.Select(f => f.I)); + Assert.Equal("Harbour sunset", manifest.Title); + Assert.Equal("Canon EOS R5", manifest.Camera!.Model); + Assert.True(manifest.Frames.All(f => f.Sha256 is null && f.Size > 0 && f.Format == ImageFileFormat.Jpeg && f.W == 48)); + foreach (var (path, bytes) in before) Assert.BytesEqual(bytes, File.ReadAllBytes(path)); + } + + [Test] + public void ReconcileIndexesNewPhotosAndFlagsMissingOnes() + { + var archive = NewArchive(); + var sequence = archive.Sequence("s"); + var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10)); + var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null); + Assert.Null(SequenceAdopter.Reconcile(sequence, manifest)); + + File.Delete(files[1]); + TestPhotos.Write(Path.Combine(sequence.Originals, "IMG_0000.JPG"), Start.AddSeconds(-10), 9); + var orphans = SequenceAdopter.FindOrphans(sequence, manifest); + Assert.SequenceEqual(["originals/IMG_0000.JPG"], orphans.UnindexedFiles); + Assert.SequenceEqual(["originals/IMG_0002.JPG"], orphans.MissingFiles.Select(f => f.File)); + + var reconciled = SequenceAdopter.Reconcile(sequence, manifest)!; + Assert.SequenceEqual(["originals/IMG_0000.JPG", "originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG", "originals/IMG_0004.JPG"], + reconciled.Frames.Select(f => f.File)); + Assert.Equal(FrameFlags.Missing, reconciled.Frames[2].Flags); + Assert.Equal(manifest.Frames[0].Id, reconciled.Frames[1].Id); + + TestPhotos.Write(files[1], Start.AddSeconds(10), 2); + var restored = SequenceAdopter.Reconcile(sequence, reconciled)!; + Assert.Equal(FrameFlags.None, restored.Frames[2].Flags); + } + + [Test] + public async Task VerifyDetectsAFlippedByteAMissingFileAndAChangedFile() + { + var archive = NewArchive(); + var sequence = archive.Sequence("s"); + var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10)); + var adopted = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null); + var hashed = new List(); + foreach (var frame in adopted.Frames) + { + var check = await FrameVerifier.CheckAsync(sequence, frame); + Assert.Equal(FrameHealth.Hashed, check.Health); + hashed.Add(check.Updated); + } + + var bytes = File.ReadAllBytes(files[0]); + bytes[bytes.Length / 2] ^= 0x01; + File.WriteAllBytes(files[0], bytes); + File.Delete(files[1]); + File.AppendAllText(files[2], "edit"); + + var health = new List(); + foreach (var frame in hashed) health.Add((await FrameVerifier.CheckAsync(sequence, frame)).Health); + Assert.SequenceEqual([FrameHealth.Corrupt, FrameHealth.Missing, FrameHealth.Changed, FrameHealth.Ok], health); + var corrupt = await FrameVerifier.CheckAsync(sequence, hashed[0]); + Assert.Equal(FrameFlags.Corrupt, corrupt.Flags); + Assert.Equal(hashed[0].Sha256, corrupt.Updated.Sha256); + } + + [Test] + public void TheIndexSnapshotIsUsedOnlyForUnchangedSidecars() + { + var archive = NewArchive(); + Directory.CreateDirectory(archive.MetaDirectory); + var stampA = new FileStamp(10, new DateTime(2026, 9, 1, 0, 0, 0, DateTimeKind.Utc)); + using (var writer = new IndexSnapshot.Writer(archive.SnapshotPath)) + { + writer.Add("a", stampA, "{\"a\":1}"u8); + writer.Add("b/ç", new FileStamp(20, stampA.ModifiedUtc), "{\"b\":2}"u8); + writer.Commit(); + } + var snapshot = IndexSnapshot.TryLoad(archive.SnapshotPath)!; + Assert.Equal(2, snapshot.Count); + Assert.BytesEqual("{\"a\":1}"u8, snapshot.TryGet("a", stampA)); + Assert.Null(snapshot.TryGet("a", stampA with { Size = 11 })); + Assert.NotNull(snapshot.TryGet("b/ç", new FileStamp(20, stampA.ModifiedUtc))); + + var bytes = File.ReadAllBytes(archive.SnapshotPath); + bytes[12] ^= 0x40; + File.WriteAllBytes(archive.SnapshotPath, bytes); + Assert.Null(IndexSnapshot.TryLoad(archive.SnapshotPath)); + } + + [Test] + public void RenamePlansAvoidTakenFoldersAndMoveIdempotently() + { + var archive = NewArchive(); + Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old a")); + Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old b")); + Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "2026-09-28 Sunset")); + var a = new RenameSubject(Guid.NewGuid(), "2026/old a", null, Start, "Canon", "Canon EOS R5", null); + var b = new RenameSubject(Guid.NewGuid(), "2026/old b", "Dawn", Start, "Canon", "Canon EOS R5", null); + var plan = RenamePlanner.Plan(Guid.NewGuid(), [a, b], "{year}/{date} {title}", "Sunset", + ["2026/old a", "2026/old b"], f => Directory.Exists(archive.FromRelative(f))); + Assert.SequenceEqual(["2026/2026-09-28 Sunset (2)", "2026/2026-09-28 Sunset (3)"], plan.Steps.Select(s => s.ToFolder)); + Assert.True(plan.Steps.All(s => s.ToTitle == "Sunset")); + Assert.Equal("Dawn", plan.Steps[1].FromTitle); + + RenamePlanner.MoveFolder(archive, plan.Steps[0]); + RenamePlanner.MoveFolder(archive, plan.Steps[0]); + Assert.False(Directory.Exists(archive.FromRelative("2026/old a"))); + Assert.True(Directory.Exists(archive.FromRelative("2026/2026-09-28 Sunset (2)"))); + + var titleOnly = RenamePlanner.Plan(Guid.NewGuid(), [b], null, "{title} morning", ["2026/old b"], _ => false); + Assert.Equal("2026/old b", titleOnly.Steps.Single().ToFolder); + Assert.Equal("Dawn morning", titleOnly.Steps.Single().ToTitle); + Assert.Empty(RenamePlanner.Plan(Guid.NewGuid(), [b], "2026/old b", null, ["2026/old b"], _ => true).Steps); + } + + [Test] + public void ThumbnailsAndProxiesAreRenderedFromTheFrame() + { + var archive = NewArchive(); + var sequence = archive.Sequence("s"); + Directory.CreateDirectory(sequence.Originals); + File.WriteAllBytes(Path.Combine(sequence.Originals, "big.jpg"), TestPhotos.Jpeg(Start, 7, width: 640, height: 480, orientation: 6)); + var frame = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null).Frames.Single(); + var renderer = new FrameImageRenderer(new ImageDecoder([])); + + var thumb = renderer.Thumbnail(sequence, frame)!; + Assert.Equal((240, 320), (thumb.Width, thumb.Height)); + Assert.Equal(DecodeTier.Native, thumb.Tier); + Assert.Equal(240, JpegDecoder.ReadInfo(thumb.Jpeg).Width); + + var proxy = renderer.Proxy(sequence, frame, new SidecarCamera { Make = "Canon", Model = "Canon EOS R5" }, new ProxyPolicy { LongEdge = 400 })!; + Assert.Equal((300, 400), (proxy.Width, proxy.Height)); + var exif = MetadataReader.Read(new MemoryStream(proxy.Jpeg), "p.jpg"); + Assert.Equal(Start, exif.CapturedAt); + Assert.Equal("Canon EOS R5", exif.Model); + Assert.Equal(1, exif.Orientation); + } +} diff --git a/tests/Titano.Core.Tests/Archiving/LeaseFileTests.cs b/tests/Titano.Core.Tests/Archiving/LeaseFileTests.cs new file mode 100644 index 00000000..7b8e98b8 --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/LeaseFileTests.cs @@ -0,0 +1,113 @@ +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class LeaseFileTests(TestContext context) +{ + private static void WriteRecord(string path, LeaseRecord record) => + File.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord)); + + [Test] + public void ALeaseIsExclusiveUntilReleased() + { + var locks = context.CreateTempDirectory(); + using (var a = LeaseFile.TryAcquire(locks, "seq-1", "server")) + { + Assert.NotNull(a); + Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client")); + using var other = LeaseFile.TryAcquire(locks, "seq-2", "client"); + Assert.NotNull(other); + } + using var b = LeaseFile.TryAcquire(locks, "seq-1", "client"); + Assert.NotNull(b); + Assert.Equal("client", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner); + } + + [Test] + public void AnExpiredLeaseOfACrashedHolderIsTakenOver() + { + var locks = context.CreateTempDirectory(); + var past = DateTimeOffset.UtcNow.AddMinutes(-10); + WriteRecord(LeaseFile.PathFor(locks, "seq-1"), new LeaseRecord("dead", "crashed", "pc", 1, past, past.AddMinutes(2))); + using var lease = LeaseFile.TryAcquire(locks, "seq-1", "survivor"); + Assert.NotNull(lease); + Assert.Equal("survivor", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner); + Assert.Count(1, Directory.GetFiles(locks)); + } + + [Test] + public async Task AHeldLeaseIsRenewedPastItsDuration() + { + var locks = context.CreateTempDirectory(); + using var lease = LeaseFile.TryAcquire(locks, "seq-1", "server", TimeSpan.FromMilliseconds(300)); + Assert.NotNull(lease); + await Task.Delay(900); + Assert.True(lease.IsHeld); + Assert.Greater((LeaseFile.Read(lease.Path)!.Expires - DateTimeOffset.UtcNow).TotalMilliseconds, 0); + Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client", TimeSpan.FromMilliseconds(300))); + } + + [Test] + public void AFreshEmptyLeaseFileIsNotStaleButAnOldOneIs() + { + var locks = context.CreateTempDirectory(); + var path = LeaseFile.PathFor(locks, "seq-1"); + File.WriteAllBytes(path, []); + Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client")); + File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddHours(-1)); + using var lease = LeaseFile.TryAcquire(locks, "seq-1", "client"); + Assert.NotNull(lease); + } + + [Test] + public void ALostLeaseIsDetectedAndNotDeletedByItsFormerHolder() + { + var locks = context.CreateTempDirectory(); + var lease = LeaseFile.TryAcquire(locks, "seq-1", "slow"); + Assert.NotNull(lease); + var now = DateTimeOffset.UtcNow; + WriteRecord(lease.Path, new LeaseRecord("thief", "other", "pc", 2, now, now.AddMinutes(2))); + Assert.False(lease.Renew()); + Assert.False(lease.IsHeld); + lease.Dispose(); + Assert.Equal("thief", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Token); + } + + [Test] + public async Task WaitingForABusyLeaseTimesOutWithItsHolder() + { + var locks = context.CreateTempDirectory(); + using var held = LeaseFile.TryAcquire(locks, "seq-1", "server"); + var error = await Assert.ThrowsAsync(() => + LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromMilliseconds(200))); + Assert.Equal("server", error.Holder!.Owner); + } + + [Test] + public async Task WaitingForALeaseSucceedsOnceItIsReleased() + { + var locks = context.CreateTempDirectory(); + var held = LeaseFile.TryAcquire(locks, "seq-1", "server")!; + var waiting = LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromSeconds(10)); + await Task.Delay(100); + held.Dispose(); + using var lease = await waiting; + Assert.Equal("client", lease.Record.Owner); + } + + [Test] + public void CleanupRemovesOnlyStaleLeasesAndItsDryRunNothing() + { + var locks = context.CreateTempDirectory(); + var past = DateTimeOffset.UtcNow.AddMinutes(-10); + WriteRecord(LeaseFile.PathFor(locks, "dead"), new LeaseRecord("t", "crashed", "pc", 1, past, past.AddMinutes(1))); + using var live = LeaseFile.TryAcquire(locks, "live", "server"); + Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: true)); + Assert.Count(2, Directory.GetFiles(locks)); + Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: false)); + Assert.SequenceEqual([LeaseFile.PathFor(locks, "live")], Directory.GetFiles(locks)); + } +} diff --git a/tests/Titano.Core.Tests/Archiving/ManifestStoreTests.cs b/tests/Titano.Core.Tests/Archiving/ManifestStoreTests.cs new file mode 100644 index 00000000..4c81061a --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/ManifestStoreTests.cs @@ -0,0 +1,157 @@ +using System.Text; +using System.Text.Json; +using Titano.Core.Archiving; +using Titano.Core.Imaging; +using Titano.Core.IO; +using Titano.Core.Model; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class ManifestStoreTests(TestContext context) +{ + private static SequenceManifest Sample() => new() + { + Id = Guid.CreateVersion7(), + Title = "Harbour sunset", + Camera = new SidecarCamera { Make = "Canon", Model = "Canon EOS R5", Serial = "0431" }, + Lens = "RF24-70mm F2.8 L IS USM", + Frames = + [ + new SidecarFrame + { + Id = Guid.CreateVersion7(), I = 0, File = "originals/2026-09-28_18-02-11_0001.CR2", Original = "IMG_0001.CR2", Size = 25_000_000, + Sha256 = new string('a', 64), CapturedAt = new DateTime(2026, 9, 28, 18, 2, 11, 250), UtcOffset = 120, + TimeSource = TimestampSource.ExifOriginal, Exposure = 1.0 / 250, FNumber = 8, Iso = 100, Focal = 24, Bias = -0.3, Wb = 0, + Orientation = 6, W = 8192, H = 5464, Format = ImageFileFormat.Cr2, PreviewOffset = 1234, PreviewLength = 567_890, + Flags = FrameFlags.Excluded, + }, + ], + Tags = ["sunset", "harbour"], + Rating = 4, + Notes = "Windy", + Status = SequenceStatus.ThumbsBuilt | SequenceStatus.Verified, + Versions = new ManifestVersions { Settings = 3, Frames = 7 }, + CreatedBy = "alberto", + Created = new DateTimeOffset(2026, 9, 28, 20, 0, 0, TimeSpan.Zero), + UpdatedAt = new DateTimeOffset(2026, 9, 29, 8, 0, 0, TimeSpan.Zero), + VerifiedAt = new DateTimeOffset(2026, 9, 30, 8, 0, 0, TimeSpan.Zero), + SessionKeys = ["card-1234abcd-20260928-180211"], + }; + + private string SidecarPath() => Path.Combine(context.CreateTempDirectory(), ".titano", "sequence.json"); + + private static string Canonical(SequenceManifest m) => JsonSerializer.Serialize(m, ArchiveJson.Default.SequenceManifest); + + [Test] + public void SequenceSidecarRoundTripsEveryField() + { + var path = SidecarPath(); + var original = Sample(); + ManifestStore.WriteSequence(path, original); + var read = ManifestStore.ReadSequence(path); + Assert.Equal(Canonical(original), Canonical(read)); + Assert.Equal(original.Frames[0], read.Frames[0] with { Extra = original.Frames[0].Extra }); + var json = File.ReadAllText(path); + foreach (var field in new[] { "\"i\":", "\"file\":", "\"sha256\":", "\"capturedAt\":", "\"fNumber\":", "\"w\":", "\"previewOffset\":", "\"flags\":" }) + Assert.Contains(field, json); + } + + [Test] + public void UnknownFieldsAndEnumValuesFromANewerBuildAreReadAndKept() + { + var path = SidecarPath(); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var id = Guid.CreateVersion7(); + var frameId = Guid.CreateVersion7(); + File.WriteAllText(path, $$""" + { + "schemaVersion": 1, "id": "{{id}}", "title": "Comet", "futureBlock": { "a": [1, 2] }, + "camera": { "make": "Nikon", "sensorTemp": 31 }, + "frames": [ { "id": "{{frameId}}", "i": 0, "file": "x.nef", "size": 10, "format": "Nef2030", "timeSource": "Atomic", + "flags": 4096, "aiScore": 0.9 } ], + "status": 1024, "versions": { "settings": 1, "frames": 2, "renders": 5 }, + } + """); + var read = ManifestStore.ReadSequence(path); + Assert.Equal(id, read.Id); + Assert.Equal(ImageFileFormat.Unknown, read.Frames[0].Format); + Assert.Equal(TimestampSource.Unknown, read.Frames[0].TimeSource); + Assert.Equal((FrameFlags)4096, read.Frames[0].Flags); + + ManifestStore.WriteSequence(path, read with { Title = "Comet 2026" }); + var json = File.ReadAllText(path); + foreach (var kept in new[] { "futureBlock", "sensorTemp", "aiScore", "renders", "4096", "1024" }) Assert.Contains(kept, json); + Assert.Equal("Comet 2026", ManifestStore.ReadSequence(path).Title); + } + + [Test] + public void ANewerSchemaIsReadableButNeverRewritten() + { + var path = SidecarPath(); + ManifestStore.WriteSequence(path, Sample()); + var newer = ManifestStore.ReadSequence(path) with { SchemaVersion = SequenceManifest.CurrentSchemaVersion + 1 }; + var before = File.ReadAllBytes(path); + Assert.Throws(() => ManifestStore.WriteSequence(path, newer)); + Assert.BytesEqual(before, File.ReadAllBytes(path)); + } + + [Test] + public void DamagedJsonIsReportedAsInvalidData() + { + var path = SidecarPath(); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, "{ \"id\": "); + Assert.Throws(() => ManifestStore.ReadSequence(path)); + File.WriteAllText(path, "{ \"title\": \"no id\" }"); + Assert.Throws(() => ManifestStore.ReadSequence(path)); + } + + [Test] + public void ACrashDuringAWriteLeavesThePreviousSidecarIntact() + { + var path = SidecarPath(); + var first = Sample(); + ManifestStore.WriteSequence(path, first); + // Simulated crash: the next version was half written to the temporary sibling and never renamed. + var torn = AtomicFile.TempPathFor(path); + File.WriteAllBytes(torn, Encoding.UTF8.GetBytes(Canonical(first with { Title = "Lost" })[..40])); + + Assert.Equal(Canonical(first), Canonical(ManifestStore.ReadSequence(path))); + ManifestStore.WriteSequence(path, first with { Title = "Second" }); + Assert.Equal("Second", ManifestStore.ReadSequence(path).Title); + Assert.FileExists(torn); + Assert.Equal(2, Directory.GetFiles(Path.GetDirectoryName(path)!).Length); + } + + [Test] + public void CreatingASidecarClaimsTheFolderOnce() + { + var path = SidecarPath(); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + var a = Sample(); + var b = Sample(); + Assert.True(ManifestStore.TryCreateSequence(path, a)); + Assert.False(ManifestStore.TryCreateSequence(path, b)); + Assert.Equal(a.Id, ManifestStore.ReadSequence(path).Id); + Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(path)!)); + } + + [Test] + public void ArchiveManifestRoundTripsAndKeepsUnknownFields() + { + var path = Path.Combine(context.CreateTempDirectory(), "archive.json"); + File.WriteAllText(path, $$""" + { "schemaVersion": 1, "id": "{{Guid.CreateVersion7()}}", "name": "Main", "created": "2026-10-02T09:00:00+00:00", + "layoutTemplates": { "folder": "{year}/{title}", "file": "{n:00000}" }, "proxyPolicy": { "longEdge": 2560, "quality": 90 }, + "syncPolicy": "mirror" } + """); + var read = ManifestStore.ReadArchive(path); + Assert.Equal("{year}/{title}", read.LayoutTemplates.Folder); + Assert.Equal(2560, read.ProxyPolicy.LongEdge); + ManifestStore.WriteArchive(path, read with { Name = "Studio" }); + Assert.Contains("syncPolicy", File.ReadAllText(path)); + Assert.Equal("Studio", ManifestStore.ReadArchive(path).Name); + } +} diff --git a/tests/Titano.Core.Tests/Archiving/TestPhotos.cs b/tests/Titano.Core.Tests/Archiving/TestPhotos.cs new file mode 100644 index 00000000..c0cb2db2 --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/TestPhotos.cs @@ -0,0 +1,51 @@ +using Titano.Core.Codecs.Jpeg; +using Titano.Core.Imaging; +using Titano.Core.Metadata; + +namespace Titano.Core.Tests.Archiving; + +/// Small real JPEGs with Exif (capture time, camera) made by the in-house encoder; each seed gives distinct bytes. +internal static class TestPhotos +{ + public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431", + int width = 48, int height = 32, int orientation = 1) + { + using var image = new ImageBuffer(width, height); + var random = new Random(seed); + for (var c = 0; c < 3; c++) + { + var plane = image.Plane(c); + var level = (float)random.NextDouble(); + for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % width) / width); + } + var exif = ExifWriter.BuildJpegApp1(new FrameMetadata + { + Make = make, + Model = model, + BodySerial = serial, + CapturedAt = captured, + ExposureTime = 1.0 / 125, + FNumber = 8, + Iso = 100, + Orientation = orientation, + }); + return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] }); + } + + public static string Write(string path, DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431") + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, Jpeg(captured, seed, make, model, serial)); + return path; + } + + /// Writes photos named IMG_0001.JPG… taken every . + public static List WriteSeries(string directory, DateTime start, int count, TimeSpan interval, int firstSeed = 1, string prefix = "IMG_", + string make = "Canon", string model = "Canon EOS R5", int firstNumber = 1) + { + var paths = new List(count); + for (var i = 0; i < count; i++) + paths.Add(Write(Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG"), start + interval * i, firstSeed + i, make, model)); + return paths; + } +} diff --git a/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs new file mode 100644 index 00000000..b360aafd --- /dev/null +++ b/tests/Titano.Core.Tests/Archiving/ThumbPackTests.cs @@ -0,0 +1,165 @@ +using Titano.Core.Archiving; +using Titano.Testing; + +namespace Titano.Core.Tests.Archiving; + +[Category("archive")] +public sealed class ThumbPackTests(TestContext context) +{ + private static byte[] Payload(int seed, int length = 300) + { + var data = new byte[length]; + new Random(seed).NextBytes(data); + return data; + } + + private static byte[] Hash(int seed) => ThumbPack.Fingerprint(null, $"originals/{seed}.jpg", seed); + + private string NewPack() => Path.Combine(context.CreateTempDirectory(), ".titano", "thumbs.pack"); + + private static void Fill(string path, int count, int seedOffset = 0) + { + using var writer = ThumbPackWriter.Open(path); + for (var i = 0; i < count; i++) writer.Append(i, 320, 213, Hash(i + seedOffset), Payload(i + seedOffset)); + } + + [Test] + public void EntriesAreReadBackByFrameIndex() + { + var path = NewPack(); + Fill(path, 100); + using var reader = ThumbPackReader.Open(path); + Assert.Equal(100, reader.Entries.Count); + Assert.BytesEqual(Payload(57), reader.Read(57)); + Assert.BytesEqual(Hash(57), reader.Entries[57].SourceHash); + Assert.Equal(320, reader.Entries[57].Width); + Assert.Null(reader.Read(100)); + } + + [Test] + public void AppendingAgainReplacesAFrameAndKeepsTheOthers() + { + var path = NewPack(); + Fill(path, 3); + using (var writer = ThumbPackWriter.Open(path)) + { + Assert.Equal(3, writer.Entries.Count); + writer.Append(1, 200, 100, Hash(99), Payload(99)); + writer.Append(3, 200, 100, Hash(3), Payload(3)); + } + using var reader = ThumbPackReader.Open(path); + Assert.Equal(4, reader.Entries.Count); + Assert.BytesEqual(Payload(99), reader.Read(1)); + Assert.BytesEqual(Payload(0), reader.Read(0)); + Assert.BytesEqual(Payload(3), reader.Read(3)); + } + + [Test] + public void AnIncompleteTailIsIgnoredAndTruncatedByTheNextWriter() + { + var path = NewPack(); + Fill(path, 2); + var committed = new FileInfo(path).Length; + // Simulated crash mid-append: an entry header and part of its payload, no index. + using (var stream = new FileStream(path, FileMode.Append)) + { + stream.Write("TENT"u8); + stream.Write(BitConverter.GetBytes(2)); + stream.Write(new byte[30]); + } + using (var reader = ThumbPackReader.Open(path)) + { + Assert.Equal(2, reader.Entries.Count); + Assert.BytesEqual(Payload(1), reader.Read(1)); + } + using (var writer = ThumbPackWriter.Open(path)) + { + Assert.Equal(committed, new FileInfo(path).Length); + writer.Append(2, 320, 213, Hash(2), Payload(2)); + } + using var after = ThumbPackReader.Open(path); + Assert.Equal(3, after.Entries.Count); + Assert.BytesEqual(Payload(2), after.Read(2)); + } + + [Test] + public void EntriesWrittenAfterTheLastIndexAreFoundWithoutIt() + { + var path = NewPack(); + Fill(path, 5); + // Simulated crash after new entries overwrote the old index but before the new index was written. + var withoutIndex = File.ReadAllBytes(path); + using (var reader = ThumbPackReader.Open(path)) + { + var end = reader.Entries.Values.Max(e => e.Offset + e.Length); + File.WriteAllBytes(path, withoutIndex[..(int)end]); + } + using var scanned = ThumbPackReader.Open(path); + Assert.Equal(5, scanned.Entries.Count); + for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), scanned.Read(i)); + } + + [Test] + public void ADamagedThumbnailReadsAsMissing() + { + var path = NewPack(); + Fill(path, 3); + long offset; + using (var reader = ThumbPackReader.Open(path)) offset = reader.Entries[1].Offset; + using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) + { + stream.Position = offset + 10; + var b = stream.ReadByte(); + stream.Position = offset + 10; + stream.WriteByte((byte)(b ^ 0xFF)); + } + using var damaged = ThumbPackReader.Open(path); + Assert.Null(damaged.Read(1)); + Assert.BytesEqual(Payload(2), damaged.Read(2)); + } + + [Test] + public void AnOpenReaderKeepsReadingWhileAWriterAppends() + { + var path = NewPack(); + Fill(path, 3); + using var reader = ThumbPackReader.Open(path); + using (var writer = ThumbPackWriter.Open(path)) + { + for (var i = 3; i < 50; i++) writer.Append(i, 320, 213, Hash(i), Payload(i, 2000)); + } + Assert.BytesEqual(Payload(0), reader.Read(0)); + Assert.BytesEqual(Payload(2), reader.Read(2)); + using var fresh = ThumbPackReader.Open(path); + Assert.Equal(50, fresh.Entries.Count); + } + + [Test] + public void CompactionDropsReplacedEntries() + { + var path = NewPack(); + for (var round = 0; round < 4; round++) Fill(path, 20, seedOffset: round * 100); + long garbage; + using (var reader = ThumbPackReader.Open(path)) garbage = reader.GarbageBytes; + Assert.Greater(garbage, 0); + var before = new FileInfo(path).Length; + ThumbPackWriter.Compact(path); + Assert.Equal(before - garbage, new FileInfo(path).Length); + using var compacted = ThumbPackReader.Open(path); + Assert.Equal(0L, compacted.GarbageBytes); + for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i)); + } + + [Test] + public void AMissingOrForeignFileReadsAsEmpty() + { + var path = NewPack(); + using (var none = ThumbPackReader.Open(path)) Assert.Equal(0, none.Entries.Count); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, "not a pack"); + using (var foreign = ThumbPackReader.Open(path)) Assert.Equal(0, foreign.Entries.Count); + Fill(path, 1); + using var rewritten = ThumbPackReader.Open(path); + Assert.Equal(1, rewritten.Entries.Count); + } +}