feat(core): archive layout, sidecars, leases and thumbnail packs

archive.json and sequence.json with source-generated JSON, forward-compatible
reading (unknown fields kept, unknown enum names tolerated) and atomic writes;
lease files with expiry and stale takeover; thumbs.pack with index, crash-safe
append and random access; folder scanning, in-place adoption, reconcile,
verify, index snapshot, rename planning and thumbnail/proxy rendering.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
Claude committed 2026-10-02 12:30:14 +00:00
1 parent 446daeae2b
commit 55895e8474
24 files changed
+2556

No files matched your search

+22
View File
@@ -5,4 +5,26 @@
<resheader name="reader"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<resheader name="writer"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<data name="Archive.Empty" xml:space="preserve"><value>No archive yet</value></data>
<data name="Archive.Error.Busy" xml:space="preserve"><value>Another Titano ({0}) is writing here. Try again shortly.</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.CopyMismatch" xml:space="preserve"><value>The copy of {0} does not match the original.</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.DuplicateLost" xml:space="preserve"><value>{0} could not be added to the sequence.</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.NewerFormat" xml:space="preserve"><value>{0} was written by a newer Titano (format {1}).</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.Offline" xml:space="preserve"><value>The archive {0} is offline.</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.ReadOnly" xml:space="preserve"><value>The archive {0} is read-only.</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.SourceChanged" xml:space="preserve"><value>{0} changed after the plan was made.</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.SourceGone" xml:space="preserve"><value>The card or folder {0} is no longer available.</value><comment>long: error sentence</comment></data>
<data name="Archive.Error.StalePlan" xml:space="preserve"><value>The archive changed after this plan was made.</value><comment>long: error sentence</comment></data>
<data name="Archive.Log.DryRun" xml:space="preserve"><value>Dry run: nothing was changed.</value><comment>long: job log line</comment></data>
<data name="Archive.Log.FolderFailed" xml:space="preserve"><value>Not indexed: {0} ({1})</value><comment>long: job log line</comment></data>
<data name="Archive.Log.FrameChanged" xml:space="preserve"><value>Changed: {0}</value><comment>long: job log line</comment></data>
<data name="Archive.Log.FrameCorrupt" xml:space="preserve"><value>Damaged: {0}</value><comment>long: job log line</comment></data>
<data name="Archive.Log.FrameMissing" xml:space="preserve"><value>Missing: {0}</value><comment>long: job log line</comment></data>
<data name="Archive.Log.ImagesDone" xml:space="preserve"><value>{0}: {1} made, {2} reused, {3} unreadable</value><comment>long: job log line</comment></data>
<data name="Archive.Log.ImportDone" xml:space="preserve"><value>{0} copied, {1} skipped, {2} failed</value><comment>long: job log line</comment></data>
<data name="Archive.Log.ImportFailed" xml:space="preserve"><value>Not copied: {0} ({1})</value><comment>long: job log line</comment></data>
<data name="Archive.Log.MaintenanceDone" xml:space="preserve"><value>{0} stale locks, {1} temporary files, {2} packs compacted, {3} duplicate flags changed</value><comment>long: job log line</comment></data>
<data name="Archive.Log.Orphans" xml:space="preserve"><value>{0} unindexed photos, {1} missing photos</value><comment>long: job log line</comment></data>
<data name="Archive.Log.ScanDone" xml:space="preserve"><value>{0} folders, {1} indexed, {2} adopted, {3} removed</value><comment>long: job log line</comment></data>
<data name="Archive.Log.SequenceMissing" xml:space="preserve"><value>Sequence not found: {0}</value><comment>long: job log line</comment></data>
<data name="Archive.Log.VerifyDone" xml:space="preserve"><value>{0} photos checked, {1} intact, {2} hashed, {3} with problems</value><comment>long: job log line</comment></data>
</root>
+22
View File
@@ -5,4 +5,26 @@
<resheader name="reader"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<resheader name="writer"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
<data name="Archive.Empty" xml:space="preserve"><value>Nessun archivio</value></data>
<data name="Archive.Error.Busy" xml:space="preserve"><value>Un altro Titano ({0}) sta scrivendo qui. Riprova tra poco.</value></data>
<data name="Archive.Error.CopyMismatch" xml:space="preserve"><value>La copia di {0} non corrisponde all'originale.</value></data>
<data name="Archive.Error.DuplicateLost" xml:space="preserve"><value>Impossibile aggiungere {0} alla sequenza.</value></data>
<data name="Archive.Error.NewerFormat" xml:space="preserve"><value>{0} è stato scritto da un Titano più recente (formato {1}).</value></data>
<data name="Archive.Error.Offline" xml:space="preserve"><value>L'archivio {0} non è raggiungibile.</value></data>
<data name="Archive.Error.ReadOnly" xml:space="preserve"><value>L'archivio {0} è di sola lettura.</value></data>
<data name="Archive.Error.SourceChanged" xml:space="preserve"><value>{0} è cambiato dopo la creazione del piano.</value></data>
<data name="Archive.Error.SourceGone" xml:space="preserve"><value>La scheda o cartella {0} non è più disponibile.</value></data>
<data name="Archive.Error.StalePlan" xml:space="preserve"><value>L'archivio è cambiato dopo la creazione del piano.</value></data>
<data name="Archive.Log.DryRun" xml:space="preserve"><value>Prova: nulla è stato modificato.</value></data>
<data name="Archive.Log.FolderFailed" xml:space="preserve"><value>Non indicizzata: {0} ({1})</value></data>
<data name="Archive.Log.FrameChanged" xml:space="preserve"><value>Modificato: {0}</value></data>
<data name="Archive.Log.FrameCorrupt" xml:space="preserve"><value>Danneggiato: {0}</value></data>
<data name="Archive.Log.FrameMissing" xml:space="preserve"><value>Mancante: {0}</value></data>
<data name="Archive.Log.ImagesDone" xml:space="preserve"><value>{0}: {1} create, {2} riusate, {3} illeggibili</value></data>
<data name="Archive.Log.ImportDone" xml:space="preserve"><value>{0} copiate, {1} saltate, {2} non riuscite</value></data>
<data name="Archive.Log.ImportFailed" xml:space="preserve"><value>Non copiata: {0} ({1})</value></data>
<data name="Archive.Log.MaintenanceDone" xml:space="preserve"><value>{0} blocchi scaduti, {1} file temporanei, {2} pacchetti compattati, {3} segni di duplicato cambiati</value></data>
<data name="Archive.Log.Orphans" xml:space="preserve"><value>{0} foto non indicizzate, {1} foto mancanti</value></data>
<data name="Archive.Log.ScanDone" xml:space="preserve"><value>{0} cartelle, {1} indicizzate, {2} adottate, {3} rimosse</value></data>
<data name="Archive.Log.SequenceMissing" xml:space="preserve"><value>Sequenza non trovata: {0}</value></data>
<data name="Archive.Log.VerifyDone" xml:space="preserve"><value>{0} foto controllate, {1} integre, {2} con impronta nuova, {3} con problemi</value></data>
</root>
+46
View File
@@ -0,0 +1,46 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Titano.Core.Archiving;
/// <summary>Source-generated metadata for every JSON file of the archive layout.</summary>
[JsonSourceGenerationOptions(
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
PropertyNameCaseInsensitive = true,
AllowTrailingCommas = true,
ReadCommentHandling = JsonCommentHandling.Skip)]
[JsonSerializable(typeof(ArchiveManifest))]
[JsonSerializable(typeof(SequenceManifest))]
[JsonSerializable(typeof(LeaseRecord))]
[JsonSerializable(typeof(IngestJournal))]
[JsonSerializable(typeof(RenameJournal))]
internal sealed partial class ArchiveJson : JsonSerializerContext;
/// <summary>Indented output for the small files people may open (<c>archive.json</c>, rename journals).</summary>
[JsonSourceGenerationOptions(
WriteIndented = true,
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(ArchiveManifest))]
[JsonSerializable(typeof(RenameJournal))]
internal sealed partial class ArchiveJsonIndented : JsonSerializerContext;
/// <summary>
/// Writes enum names and reads unknown names (written by a newer build) or numbers as the default value instead of
/// failing, so a sidecar from a newer Titano is still readable.
/// </summary>
public sealed class TolerantEnumConverter<TEnum> : JsonConverter<TEnum>
where TEnum : struct, Enum
{
public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.String && Enum.TryParse<TEnum>(reader.GetString(), ignoreCase: true, out var value)
&& Enum.IsDefined(value))
return value;
if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) reader.Skip();
return default;
}
public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToString());
}
+104
View File
@@ -0,0 +1,104 @@
using System.Globalization;
using Titano.Core.IO;
namespace Titano.Core.Archiving;
/// <summary>Names of the fixed files and folders of the archive layout (BRIEF §7).</summary>
public static class ArchiveLayout
{
/// <summary>Metadata folder at the archive root and inside every sequence folder.</summary>
public const string MetaFolder = ".titano";
public const string ArchiveManifestFile = "archive.json";
public const string IndexSnapshotFile = "index-snapshot.bin";
public const string LocksFolder = "locks";
/// <summary>Verified copies waiting to be committed; on the archive volume so commits are renames.</summary>
public const string StagingFolder = "staging";
/// <summary>Resumable plans of mass operations (rename).</summary>
public const string JournalFolder = "journal";
public const string OriginalsFolder = "originals";
public const string ProxiesFolder = "proxies";
public const string ExportsFolder = "exports";
public const string SequenceManifestFile = "sequence.json";
public const string AnalysisFile = "analysis.json";
public const string ProjectFile = "project.json";
public const string ThumbsPackFile = "thumbs.pack";
/// <summary>Moves of an ingest batch that were started but not yet recorded in <see cref="SequenceManifestFile"/>.</summary>
public const string IngestJournalFile = "ingest.pending.json";
/// <summary>Folder names inside a sequence folder that never hold a sequence of their own.</summary>
public static bool IsReservedFolder(string name) =>
name.Equals(MetaFolder, StringComparison.OrdinalIgnoreCase)
|| name.Equals(OriginalsFolder, StringComparison.OrdinalIgnoreCase)
|| name.Equals(ProxiesFolder, StringComparison.OrdinalIgnoreCase)
|| name.Equals(ExportsFolder, StringComparison.OrdinalIgnoreCase);
/// <summary><c>0001.jpg</c> for frame index 0: proxies are numbered from 1 like the frames of a render.</summary>
public static string ProxyFileName(int index) => (index + 1).ToString("0000", CultureInfo.InvariantCulture) + ".jpg";
}
/// <summary>Absolute paths of one archive root.</summary>
public sealed class ArchivePaths
{
public ArchivePaths(string root)
{
ArgumentException.ThrowIfNullOrWhiteSpace(root);
Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)));
MetaDirectory = Path.Combine(Root, ArchiveLayout.MetaFolder);
}
public string Root { get; }
public string MetaDirectory { get; }
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.ArchiveManifestFile);
public string SnapshotPath => Path.Combine(MetaDirectory, ArchiveLayout.IndexSnapshotFile);
public string LocksDirectory => Path.Combine(MetaDirectory, ArchiveLayout.LocksFolder);
public string StagingDirectory => Path.Combine(MetaDirectory, ArchiveLayout.StagingFolder);
public string JournalDirectory => Path.Combine(MetaDirectory, ArchiveLayout.JournalFolder);
/// <summary>Paths of the sequence stored in <paramref name="relativeFolder"/> (forward slashes).</summary>
public SequencePaths Sequence(string relativeFolder) => new(this, relativeFolder);
/// <summary>Archive-relative path with forward slashes, as stored in the index.</summary>
public string ToRelative(string path) => LongPath.ToPortableRelative(Root, path);
public string FromRelative(string relative) => LongPath.FromPortableRelative(Root, relative);
}
/// <summary>Absolute paths of one sequence folder.</summary>
public sealed class SequencePaths
{
internal SequencePaths(ArchivePaths archive, string relativeFolder)
{
ArgumentNullException.ThrowIfNull(relativeFolder);
RelativeFolder = relativeFolder.Replace('\\', '/').Trim('/');
Folder = archive.FromRelative(RelativeFolder);
MetaDirectory = Path.Combine(Folder, ArchiveLayout.MetaFolder);
}
/// <summary>Folder relative to the archive root, forward slashes; empty when the root itself holds the photos (adopted folder).</summary>
public string RelativeFolder { get; }
public string Folder { get; }
public string MetaDirectory { get; }
public string Originals => Path.Combine(Folder, ArchiveLayout.OriginalsFolder);
public string Proxies => Path.Combine(Folder, ArchiveLayout.ProxiesFolder);
public string Exports => Path.Combine(Folder, ArchiveLayout.ExportsFolder);
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.SequenceManifestFile);
public string AnalysisPath => Path.Combine(MetaDirectory, ArchiveLayout.AnalysisFile);
public string ProjectPath => Path.Combine(MetaDirectory, ArchiveLayout.ProjectFile);
public string ThumbsPackPath => Path.Combine(MetaDirectory, ArchiveLayout.ThumbsPackFile);
public string IngestJournalPath => Path.Combine(MetaDirectory, ArchiveLayout.IngestJournalFile);
public string Proxy(int index) => Path.Combine(Proxies, ArchiveLayout.ProxyFileName(index));
/// <summary>Absolute path of a frame file given its sidecar path (relative to the sequence folder).</summary>
public string File(string sequenceRelative) => LongPath.FromPortableRelative(Folder, sequenceRelative);
/// <summary>Archive-relative path of a frame file (as stored in the frames table).</summary>
public string ArchiveRelative(string sequenceRelative) =>
RelativeFolder.Length == 0 ? sequenceRelative : RelativeFolder + "/" + sequenceRelative;
}
@@ -0,0 +1,49 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Titano.Core.Naming;
namespace Titano.Core.Archiving;
/// <summary>
/// <c>.titano/archive.json</c> at the archive root (BRIEF §7). Fields this build does not know are kept in
/// <see cref="Extra"/> and written back unchanged.
/// </summary>
public sealed record ArchiveManifest
{
public const int CurrentSchemaVersion = 1;
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
public required Guid Id { get; init; }
public required string Name { get; init; }
public DateTimeOffset Created { get; init; }
public LayoutTemplates LayoutTemplates { get; init; } = new();
public ProxyPolicy ProxyPolicy { get; init; } = new();
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>Naming templates used when frames enter the archive (BRIEF §7.1).</summary>
public sealed record LayoutTemplates
{
public string Folder { get; init; } = TemplateDefaults.Folder;
public string File { get; init; } = TemplateDefaults.File;
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>Proxy generation policy (BRIEF §8.5).</summary>
public sealed record ProxyPolicy
{
public const int Disabled = -1;
public const int FullSize = 0;
/// <summary>Long edge in pixels: 1280, 1920 (default) or 2560; <see cref="FullSize"/> or <see cref="Disabled"/>.</summary>
public int LongEdge { get; init; } = 1920;
public int Quality { get; init; } = 92;
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
@@ -0,0 +1,77 @@
using Titano.Core.Imaging;
namespace Titano.Core.Archiving;
/// <summary>A folder that holds a sequence: it has a sidecar, or photos to adopt.</summary>
/// <param name="RelativeFolder">Archive-relative, forward slashes; empty for the root.</param>
/// <param name="Sidecar">Stamp of <c>.titano/sequence.json</c>, null when it has none yet.</param>
/// <param name="Photos">Stamp of the photo directory (its time changes when files are added or removed).</param>
public sealed record SequenceFolder(string RelativeFolder, FileStamp? Sidecar, FileStamp? Photos);
/// <summary>
/// Walks an archive tree (BRIEF §7). No file-system watcher is involved: callers compare the stamps with those of
/// the previous scan. Hidden folders, symbolic links and the reserved sequence subfolders are not entered.
/// </summary>
public static class ArchiveScanner
{
private const int MaxDepth = 16;
private static readonly EnumerationOptions Listing = new()
{
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System,
RecurseSubdirectories = false,
};
/// <summary>Every sequence folder under the root, ordered by relative path (ordinal) so job checkpoints are stable.</summary>
public static IReadOnlyList<SequenceFolder> FindSequenceFolders(ArchivePaths archive, CancellationToken cancellationToken = default)
{
var found = new List<SequenceFolder>();
var pending = new Stack<(string Path, int Depth)>();
pending.Push((archive.Root, 0));
while (pending.Count > 0)
{
cancellationToken.ThrowIfCancellationRequested();
var (path, depth) = pending.Pop();
var relative = archive.ToRelative(path);
var sequence = archive.Sequence(relative == "." ? string.Empty : relative);
var sidecar = FileStamp.Of(sequence.ManifestPath);
var photoDirectory = PhotoDirectory(sequence);
if (sidecar is not null || EnumeratePhotos(photoDirectory).Any())
found.Add(new SequenceFolder(sequence.RelativeFolder, sidecar, FileStamp.Of(photoDirectory)));
if (depth >= MaxDepth) continue;
foreach (var child in Directory.EnumerateDirectories(path, "*", Listing))
{
var name = Path.GetFileName(child);
if (!IsHidden(child, name) && !ArchiveLayout.IsReservedFolder(name)) pending.Push((child, depth + 1));
}
}
found.Sort((a, b) => string.CompareOrdinal(a.RelativeFolder, b.RelativeFolder));
return found;
}
/// <summary><c>originals/</c> when the folder follows the layout, otherwise the folder itself (adopted in place).</summary>
public static string PhotoDirectory(SequencePaths sequence) => Directory.Exists(sequence.Originals) ? sequence.Originals : sequence.Folder;
/// <summary>Photos of a sequence folder as sidecar paths (relative to the sequence folder), ordinal order.</summary>
public static IReadOnlyList<string> ListPhotos(SequencePaths sequence)
{
var directory = PhotoDirectory(sequence);
return EnumeratePhotos(directory)
.Select(path => Path.GetRelativePath(sequence.Folder, path).Replace('\\', '/'))
.Order(StringComparer.Ordinal)
.ToList();
}
/// <summary>True for files Titano imports as photos (AppleDouble <c>._</c> companions excluded).</summary>
public static bool IsPhoto(string fileName) =>
!fileName.StartsWith("._", StringComparison.Ordinal) && ImageFormatInfo.IsSupportedExtension(fileName);
private static IEnumerable<string> EnumeratePhotos(string directory) =>
Directory.Exists(directory)
? Directory.EnumerateFiles(directory, "*", Listing).Where(f => IsPhoto(Path.GetFileName(f)) && !IsHidden(f, Path.GetFileName(f)))
: [];
private static bool IsHidden(string path, string name) =>
name.StartsWith('.') || (OperatingSystem.IsWindows() && File.GetAttributes(path).HasFlag(FileAttributes.Hidden));
}
@@ -0,0 +1,55 @@
using Titano.Core.Localization;
namespace Titano.Core.Archiving;
/// <summary>The archive cannot be reached (or written) right now; nothing was changed.</summary>
public sealed class ArchiveOfflineException(string root, ArchiveAvailability availability)
: IOException(Loc.L(availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", root))
{
public string Root { get; } = root;
public ArchiveAvailability Availability { get; } = availability;
}
public enum ArchiveAvailability
{
Online,
/// <summary>Readable but not writable (read-only share, write-protected disk): browse only.</summary>
ReadOnly,
/// <summary>Root unreachable (unplugged drive, disconnected share) or no longer an archive.</summary>
Offline,
}
/// <summary>Offline detection for archives on removable and network volumes (BRIEF §7).</summary>
public static class ArchiveVolume
{
/// <summary>Throws <see cref="ArchiveOfflineException"/> unless the archive is online (writable).</summary>
public static void EnsureWritable(ArchivePaths archive)
{
var availability = Probe(archive);
if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(archive.Root, availability);
}
/// <summary>Checks that <c>archive.json</c> is reachable and that the metadata folder accepts writes.</summary>
public static ArchiveAvailability Probe(ArchivePaths archive)
{
try
{
if (!File.Exists(archive.ManifestPath)) return ArchiveAvailability.Offline;
var probe = Path.Combine(archive.MetaDirectory, ".probe-" + Guid.NewGuid().ToString("N")[..8]);
using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1, FileOptions.DeleteOnClose))
{
}
return ArchiveAvailability.Online;
}
catch (UnauthorizedAccessException)
{
return ArchiveAvailability.ReadOnly;
}
catch (IOException)
{
return File.Exists(archive.ManifestPath) ? ArchiveAvailability.ReadOnly : ArchiveAvailability.Offline;
}
}
}
+44
View File
@@ -0,0 +1,44 @@
using System.Security.Cryptography;
namespace Titano.Core.Archiving;
/// <summary>SHA-256 of files, streamed (originals can be hundreds of megabytes).</summary>
public static class ContentHash
{
internal const int BufferSize = 1 << 20;
/// <summary>Lower-case hex SHA-256 of the file.</summary>
/// <param name="progress">Receives the number of bytes read since the previous call.</param>
public static async Task<string> Sha256Async(string path, Action<long>? progress = null, CancellationToken cancellationToken = default)
{
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, BufferSize,
FileOptions.Asynchronous | FileOptions.SequentialScan);
return await Sha256Async(stream, progress, cancellationToken).ConfigureAwait(false);
}
public static async Task<string> Sha256Async(Stream stream, Action<long>? progress = null, CancellationToken cancellationToken = default)
{
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var buffer = new byte[BufferSize];
int read;
while ((read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
{
hash.AppendData(buffer, 0, read);
progress?.Invoke(read);
}
return Convert.ToHexStringLower(hash.GetHashAndReset());
}
}
/// <summary>Size and modification time of a file, the cheap change detector of incremental scans (BRIEF §7).</summary>
public readonly record struct FileStamp(long Size, DateTime ModifiedUtc)
{
/// <summary>The stamp of a file or directory, or null when it does not exist.</summary>
public static FileStamp? Of(string path)
{
var file = new FileInfo(path);
if (file.Exists) return new FileStamp(file.Length, file.LastWriteTimeUtc);
var directory = new DirectoryInfo(path);
return directory.Exists ? new FileStamp(0, directory.LastWriteTimeUtc) : null;
}
}
@@ -0,0 +1,97 @@
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Imaging;
using Titano.Core.Metadata;
namespace Titano.Core.Archiving;
/// <summary>A JPEG made from a frame.</summary>
public sealed record EncodedImage(byte[] Jpeg, int Width, int Height, DecodeTier Tier);
/// <summary>
/// Makes thumbnails and proxies of frames (BRIEF §8.5): the embedded preview or the managed JPEG decoder first, then
/// the host's decode providers (WIC on Windows); orientation applied, resampled in linear light, encoded in-house.
/// </summary>
public sealed class FrameImageRenderer(ImageDecoder decoder)
{
public const int ThumbnailLongEdge = 320;
public const int ThumbnailQuality = 80;
/// <summary>A 320 px q80 thumbnail, or null when no decoder can read the frame.</summary>
public EncodedImage? Thumbnail(SequencePaths sequence, SidecarFrame frame, CancellationToken cancellationToken = default)
{
using var decoded = Decode(sequence.File(frame.File), frame, ThumbnailLongEdge, DecodePurpose.Thumbnail, preferPreview: true, cancellationToken);
if (decoded is null) return null;
using var small = Resampler.Thumbnail(decoded.Pixels, ThumbnailLongEdge);
return Encode(small, new JpegEncodeOptions { Quality = ThumbnailQuality }, decoded.Tier);
}
/// <summary>A proxy at the policy's long edge and quality with capture time and camera in its Exif; null when undecodable.</summary>
public EncodedImage? Proxy(SequencePaths sequence, SidecarFrame frame, SidecarCamera? camera, ProxyPolicy policy, CancellationToken cancellationToken = default)
{
if (policy.LongEdge == ProxyPolicy.Disabled) throw new InvalidOperationException("Proxies are disabled for this archive.");
using var decoded = Decode(sequence.File(frame.File), frame, policy.LongEdge, DecodePurpose.Proxy, preferPreview: false, cancellationToken);
if (decoded is null) return null;
var source = decoded.Pixels;
var (w, h) = policy.LongEdge == ProxyPolicy.FullSize ? (source.Width, source.Height) : Resampler.FitWithin(source.Width, source.Height, policy.LongEdge);
using var resized = Resampler.Resize(source, w, h);
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata
{
Make = camera?.Make,
Model = camera?.Model,
BodySerial = camera?.Serial,
CapturedAt = frame.CapturedAt,
UtcOffsetMinutes = frame.UtcOffset,
ExposureTime = frame.Exposure,
FNumber = frame.FNumber,
Iso = frame.Iso,
FocalLength = frame.Focal,
ExposureBias = frame.Bias,
Software = "Titano",
});
// The encoder takes the APP1 payload; ExifWriter returns the whole segment (marker and length first).
return Encode(resized, new JpegEncodeOptions { Quality = policy.Quality, App1Segments = [exif[4..]] }, decoded.Tier);
}
private static EncodedImage Encode(ImageBuffer pixels, JpegEncodeOptions options, DecodeTier tier) =>
new(JpegEncoder.Encode(pixels, options), pixels.Width, pixels.Height, tier);
private DecodedFrame? Decode(string path, SidecarFrame frame, int maxLongEdge, DecodePurpose purpose, bool preferPreview, CancellationToken cancellationToken)
{
if (frame.Format == ImageFileFormat.Jpeg) return DecodeJpeg(File.ReadAllBytes(path), maxLongEdge, frame.Orientation, DecodeTier.Native);
if (preferPreview && ReadPreview(path, frame) is { } preview) return DecodeJpeg(preview, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview);
var request = new DecodeRequest(path, System.Math.Max(0, maxLongEdge), purpose, ApplyOrientation: true, AllowPreview: true);
if (decoder.Decode(request, cancellationToken) is { } decoded) return new DecodedFrame(decoded.Pixels, decoded.Tier);
return ReadPreview(path, frame) is { } fallback ? DecodeJpeg(fallback, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview) : null;
}
private static DecodedFrame? DecodeJpeg(byte[] data, int maxLongEdge, int orientation, DecodeTier tier)
{
ImageBuffer decoded;
try
{
decoded = JpegDecoder.DecodeToLinear(data, new JpegDecodeOptions { MaxLongEdge = System.Math.Max(0, maxLongEdge) }, expandGray: true);
}
catch (InvalidDataException)
{
return null;
}
if (orientation is <= 1 or > 8) return new DecodedFrame(decoded, tier);
using (decoded) return new DecodedFrame(PixelConvert.ApplyOrientation(decoded, orientation), tier);
}
private static byte[]? ReadPreview(string path, SidecarFrame frame)
{
if (frame.PreviewOffset is not { } offset || frame.PreviewLength is not ({ } length and > 0 and < int.MaxValue)) return null;
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 4096, FileOptions.RandomAccess);
if (offset < 0 || offset + length > stream.Length) return null;
var data = new byte[length];
stream.Position = offset;
stream.ReadExactly(data);
return data;
}
private sealed record DecodedFrame(ImageBuffer Pixels, DecodeTier Tier) : IDisposable
{
public void Dispose() => Pixels.Dispose();
}
}
@@ -0,0 +1,54 @@
using Titano.Core.Model;
namespace Titano.Core.Archiving;
public enum FrameHealth
{
/// <summary>Same size and SHA-256 as recorded.</summary>
Ok,
/// <summary>No hash was recorded (adopted file); it has been computed now.</summary>
Hashed,
Missing,
/// <summary>A different size: the file was replaced or edited.</summary>
Changed,
/// <summary>Same size, different content: bit rot or a damaged copy.</summary>
Corrupt,
}
/// <param name="Sha256">The hash computed now (null when the file is missing or changed size).</param>
public sealed record FrameCheck(SidecarFrame Frame, FrameHealth Health, string? Sha256)
{
/// <summary>Frame flags after this check: <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> set or cleared.</summary>
public FrameFlags Flags => Health switch
{
FrameHealth.Missing => (Frame.Flags & ~FrameFlags.Corrupt) | FrameFlags.Missing,
FrameHealth.Changed or FrameHealth.Corrupt => (Frame.Flags & ~FrameFlags.Missing) | FrameFlags.Corrupt,
_ => Frame.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt),
};
/// <summary>The sidecar record after this check (flags, and the hash of a newly hashed file).</summary>
public SidecarFrame Updated => Frame with { Flags = Flags, Sha256 = Health == FrameHealth.Hashed ? Sha256 : Frame.Sha256 };
public bool ChangesRecord => Updated != Frame;
}
/// <summary>Re-hashes originals against their sidecar records (BRIEF §12.5 Verify).</summary>
public static class FrameVerifier
{
public static async Task<FrameCheck> CheckAsync(SequencePaths sequence, SidecarFrame frame, Action<long>? bytesRead = null,
CancellationToken cancellationToken = default)
{
var info = new FileInfo(sequence.File(frame.File));
if (!info.Exists) return new FrameCheck(frame, FrameHealth.Missing, null);
if (info.Length != frame.Size) return new FrameCheck(frame, FrameHealth.Changed, null);
var sha = await ContentHash.Sha256Async(info.FullName, bytesRead, cancellationToken).ConfigureAwait(false);
var health = frame.Sha256 is null ? FrameHealth.Hashed
: sha.Equals(frame.Sha256, StringComparison.OrdinalIgnoreCase) ? FrameHealth.Ok
: FrameHealth.Corrupt;
return new FrameCheck(frame, health, sha);
}
}
+127
View File
@@ -0,0 +1,127 @@
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;
using Titano.Core.IO;
namespace Titano.Core.Archiving;
/// <summary>
/// <c>.titano/index-snapshot.bin</c> (BRIEF §7): every sidecar of the archive in one file, so a rebuild reads one file
/// sequentially instead of thousands of small ones over a share. Optional and never authoritative: an entry is used
/// only when its sidecar still has the recorded size and time, and a file whose trailing SHA-256 does not match is
/// ignored entirely.
/// </summary>
/// <remarks>Layout: "TISN", version (u16), reserved (u16), then per sidecar: folder length (u16), UTF-8 folder,
/// size (i64), modified UTC ticks (i64), length (i32), sidecar bytes; finally the SHA-256 of everything before it.</remarks>
public sealed class IndexSnapshot
{
private const ushort Version = 1;
private static ReadOnlySpan<byte> Magic => "TISN"u8;
private readonly Dictionary<string, (FileStamp Stamp, byte[] Bytes)> _entries;
private IndexSnapshot(Dictionary<string, (FileStamp, byte[])> entries) => _entries = entries;
public int Count => _entries.Count;
/// <summary>The snapshot at <paramref name="path"/>, or null when it is absent, damaged or of another version.</summary>
public static IndexSnapshot? TryLoad(string path)
{
byte[] data;
try
{
if (!File.Exists(path)) return null;
data = File.ReadAllBytes(path);
}
catch (IOException)
{
return null;
}
if (data.Length < 8 + 32 || !data.AsSpan(0, 4).SequenceEqual(Magic) || BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan(4)) != Version)
return null;
var body = data.AsSpan(0, data.Length - 32);
if (!SHA256.HashData(body).AsSpan().SequenceEqual(data.AsSpan(data.Length - 32))) return null;
var entries = new Dictionary<string, (FileStamp, byte[])>(StringComparer.Ordinal);
var position = 8;
while (position < body.Length)
{
if (position + 2 > body.Length) return null;
var folderLength = BinaryPrimitives.ReadUInt16LittleEndian(body[position..]);
position += 2;
if (position + folderLength + 20 > body.Length) return null;
var folder = Encoding.UTF8.GetString(body.Slice(position, folderLength));
position += folderLength;
var size = BinaryPrimitives.ReadInt64LittleEndian(body[position..]);
var ticks = BinaryPrimitives.ReadInt64LittleEndian(body[(position + 8)..]);
var length = BinaryPrimitives.ReadInt32LittleEndian(body[(position + 16)..]);
position += 20;
if (length < 0 || ticks < 0 || ticks > DateTime.MaxValue.Ticks || position + length > body.Length) return null;
entries[folder] = (new FileStamp(size, new DateTime(ticks, DateTimeKind.Utc)), body.Slice(position, length).ToArray());
position += length;
}
return new IndexSnapshot(entries);
}
/// <summary>The recorded sidecar bytes of <paramref name="folder"/> when its current stamp still matches.</summary>
public byte[]? TryGet(string folder, FileStamp current) =>
_entries.TryGetValue(folder, out var entry) && entry.Stamp == current ? entry.Bytes : null;
/// <summary>Streams a new snapshot to a temporary file; <see cref="Commit"/> replaces the old one atomically.</summary>
public sealed class Writer : IDisposable
{
private readonly string _path;
private readonly string _temp;
private readonly FileStream _stream;
private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
private bool _committed;
public Writer(string path)
{
_path = path;
_temp = AtomicFile.TempPathFor(path);
_stream = new FileStream(_temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16);
Span<byte> header = stackalloc byte[8];
header.Clear();
Magic.CopyTo(header);
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
Write(header);
}
public void Add(string folder, FileStamp stamp, ReadOnlySpan<byte> sidecar)
{
var name = Encoding.UTF8.GetBytes(folder);
Span<byte> head = stackalloc byte[2];
BinaryPrimitives.WriteUInt16LittleEndian(head, checked((ushort)name.Length));
Write(head);
Write(name);
Span<byte> fields = stackalloc byte[20];
BinaryPrimitives.WriteInt64LittleEndian(fields, stamp.Size);
BinaryPrimitives.WriteInt64LittleEndian(fields[8..], stamp.ModifiedUtc.Ticks);
BinaryPrimitives.WriteInt32LittleEndian(fields[16..], sidecar.Length);
Write(fields);
Write(sidecar);
}
public void Commit()
{
_stream.Write(_hash.GetHashAndReset());
_stream.Dispose();
AtomicFile.Commit(_temp, _path);
_committed = true;
}
public void Dispose()
{
_stream.Dispose();
_hash.Dispose();
if (!_committed && File.Exists(_temp)) File.Delete(_temp);
}
private void Write(ReadOnlySpan<byte> data)
{
_hash.AppendData(data);
_stream.Write(data);
}
}
}
@@ -0,0 +1,52 @@
using System.Text.Json;
using Titano.Core.IO;
namespace Titano.Core.Archiving;
/// <summary>
/// Write-ahead record of an ingest batch (<c>.titano/ingest.pending.json</c>): the frames whose files are about to be
/// moved into the sequence folder. Written before the moves and removed once <c>sequence.json</c> lists them, so a
/// crash in between is repaired by <see cref="Recover"/> instead of leaving unindexed originals behind.
/// </summary>
public sealed record IngestJournal(IReadOnlyList<SidecarFrame> Frames)
{
public static void Write(SequencePaths sequence, IngestJournal journal)
{
Directory.CreateDirectory(sequence.MetaDirectory);
AtomicFile.WriteAllBytes(sequence.IngestJournalPath, JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJson.Default.IngestJournal));
}
public static void Delete(SequencePaths sequence)
{
if (File.Exists(sequence.IngestJournalPath)) File.Delete(sequence.IngestJournalPath);
}
/// <summary>
/// Completes an interrupted batch: frames whose file reached the sequence folder (same size) but are missing from
/// <paramref name="manifest"/> are added in capture order. Returns the manifest to write, or null when nothing has
/// to change. The journal is left in place; <see cref="Delete"/> it once the result is written.
/// </summary>
public static SequenceManifest? Recover(SequencePaths sequence, SequenceManifest manifest)
{
if (!File.Exists(sequence.IngestJournalPath)) return null;
IngestJournal? journal;
try
{
journal = JsonSerializer.Deserialize(File.ReadAllBytes(sequence.IngestJournalPath), ArchiveJson.Default.IngestJournal);
}
catch (JsonException)
{
journal = null; // torn while being written: nothing was moved yet
}
if (journal is null) return null;
var known = manifest.Frames.Select(f => f.Id).ToHashSet();
var moved = journal.Frames.Where(f => !known.Contains(f.Id) && FileHasSize(sequence.File(f.File), f.Size)).ToList();
return moved.Count == 0 ? null : SequenceFrames.Insert(manifest, moved);
}
private static bool FileHasSize(string path, long size)
{
var info = new FileInfo(path);
return info.Exists && info.Length == size;
}
}
+241
View File
@@ -0,0 +1,241 @@
using System.Text.Json;
using Titano.Core.Diagnostics;
using Titano.Core.Localization;
namespace Titano.Core.Archiving;
/// <summary>Content of a lease file: who holds it and until when.</summary>
public sealed record LeaseRecord(string Token, string Owner, string Host, int ProcessId, DateTimeOffset Acquired, DateTimeOffset Expires);
/// <summary>The lease is held by someone else and did not free up in time.</summary>
public sealed class LeaseUnavailableException(string name, LeaseRecord? holder)
: IOException(Loc.L("Archive.Error.Busy", holder?.Owner ?? "?"))
{
public string Name { get; } = name;
public LeaseRecord? Holder { get; } = holder;
}
/// <summary>
/// Cross-process exclusive lease stored as <c>.titano/locks/&lt;name&gt;.lease</c> (BRIEF §7), for writers that share
/// an archive over a network (server and client). Creation uses create-new semantics, which SMB and local file
/// systems both make atomic. A lease past its expiry is stale (its holder crashed or lost the share) and is taken
/// over by renaming it aside, which only one contender can do. The holder renews it in the background while held.
/// </summary>
public sealed class LeaseFile : IDisposable
{
public static readonly TimeSpan DefaultDuration = TimeSpan.FromMinutes(2);
private const string Extension = ".lease";
private const string StaleExtension = ".stale";
private readonly Lock _gate = new();
private readonly Timer _renewal;
private LeaseRecord _record;
private bool _released;
private LeaseFile(string path, string name, LeaseRecord record, TimeSpan duration)
{
Path = path;
Name = name;
_record = record;
Duration = duration;
var period = duration / 3;
_renewal = new Timer(_ => Renew(), null, period, period);
}
public string Path { get; }
public string Name { get; }
public TimeSpan Duration { get; }
public LeaseRecord Record => _record;
/// <summary>False once a renewal found the lease taken over (it expired while this process was stalled).</summary>
public bool IsHeld { get; private set; } = true;
/// <summary>Takes the lease when it is free or stale; null when someone else holds it.</summary>
public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null)
{
var length = duration ?? DefaultDuration;
var path = PathFor(locksDirectory, name);
Directory.CreateDirectory(locksDirectory);
for (var attempt = 0; attempt < 3; attempt++)
{
var now = DateTimeOffset.UtcNow;
var record = new LeaseRecord(Guid.NewGuid().ToString("N"), owner, Environment.MachineName, Environment.ProcessId, now, now + length);
if (TryCreate(path, record)) return new LeaseFile(path, name, record, length);
if (!IsStale(path, now)) return null;
BreakStale(path, now);
}
return null;
}
/// <summary>Waits up to <paramref name="wait"/> for the lease, polling with back-off.</summary>
/// <exception cref="LeaseUnavailableException">Still held by someone else after <paramref name="wait"/>.</exception>
public static async Task<LeaseFile> AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait,
TimeSpan? duration = null, CancellationToken cancellationToken = default)
{
var deadline = DateTime.UtcNow + wait;
var delay = TimeSpan.FromMilliseconds(20);
while (true)
{
if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease;
if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name)));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, 500));
}
}
/// <summary>The current holder of a lease, or null when free or unreadable.</summary>
public static LeaseRecord? Read(string path)
{
try
{
return JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.LeaseRecord);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
{
return null;
}
}
/// <summary>Removes every stale lease in <paramref name="locksDirectory"/> (maintenance); returns their names.</summary>
public static IReadOnlyList<string> CleanupStale(string locksDirectory, bool dryRun)
{
if (!Directory.Exists(locksDirectory)) return [];
var now = DateTimeOffset.UtcNow;
var removed = new List<string>();
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + Extension))
{
if (!IsStale(path, now)) continue;
removed.Add(System.IO.Path.GetFileNameWithoutExtension(path));
if (!dryRun) BreakStale(path, now);
}
if (!dryRun)
{
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + StaleExtension))
{
if (File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime) TryDelete(path);
}
}
return removed;
}
/// <summary>Extends the expiry; false when the lease is no longer ours.</summary>
public bool Renew()
{
lock (_gate)
{
if (_released || !IsHeld) return false;
if (Read(Path)?.Token != _record.Token)
{
IsHeld = false;
Log.Warning("archive", $"Lease {Name} was taken over by another writer");
return false;
}
var renewed = _record with { Expires = DateTimeOffset.UtcNow + Duration };
try
{
// In-place rewrite: a rename would open a window in which the lease looks free.
using var fs = new FileStream(Path, FileMode.Truncate, FileAccess.Write, FileShare.Read);
fs.Write(JsonSerializer.SerializeToUtf8Bytes(renewed, ArchiveJson.Default.LeaseRecord));
fs.Flush(flushToDisk: true);
_record = renewed;
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Runs on a timer thread: an escaping exception would end the process.
Log.Warning("archive", $"Lease {Name} could not be renewed", ex);
return false;
}
}
}
public void Dispose()
{
_renewal.Dispose();
lock (_gate)
{
if (_released) return;
_released = true;
if (!IsHeld || Read(Path)?.Token != _record.Token) return;
try
{
File.Delete(Path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// It expires on its own.
Log.Warning("archive", $"Lease {Name} could not be released", ex);
}
}
}
internal static string PathFor(string locksDirectory, string name) => System.IO.Path.Combine(locksDirectory, name + Extension);
private static bool TryCreate(string path, LeaseRecord record)
{
try
{
using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
fs.Flush(flushToDisk: true);
return true;
}
catch (IOException) when (File.Exists(path))
{
return false;
}
}
/// <summary>Expired, or unreadable for longer than a lease lasts (a holder that crashed while creating it).</summary>
private static bool IsStale(string path, DateTimeOffset now)
{
if (Read(path) is { } record) return record.Expires < now;
try
{
return File.Exists(path) && File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime;
}
catch (IOException)
{
return false;
}
}
private static void BreakStale(string path, DateTimeOffset now)
{
var aside = path + "." + Guid.NewGuid().ToString("N")[..8] + StaleExtension;
try
{
File.Move(path, aside);
}
catch (IOException)
{
return; // another contender broke it first
}
if (IsStale(aside, now))
{
TryDelete(aside);
return;
}
// A contender renewed or re-created the lease between our check and the rename: give it back.
try
{
File.Move(aside, path, overwrite: false);
}
catch (IOException)
{
TryDelete(aside);
}
}
private static void TryDelete(string path)
{
try
{
File.Delete(path);
}
catch (IOException)
{
// A leftover file is harmless; maintenance removes it.
}
}
}
@@ -0,0 +1,84 @@
using System.Text.Json;
using Titano.Core.IO;
using Titano.Core.Localization;
namespace Titano.Core.Archiving;
/// <summary>A sidecar written by a newer Titano: it can be read, but this build refuses to rewrite it.</summary>
public sealed class NewerFormatException(string path, int version)
: InvalidOperationException(Loc.L("Archive.Error.NewerFormat", System.IO.Path.GetFileName(path), version))
{
public string Path { get; } = path;
public int Version { get; } = version;
}
/// <summary>
/// Reads and writes the JSON files of the layout. Reading tolerates unknown fields and enum values; every write
/// goes through <see cref="AtomicFile"/> (temp → fsync → rename), so a crash leaves the previous file intact.
/// </summary>
public static class ManifestStore
{
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
public static ArchiveManifest ReadArchive(string path) => Parse(File.ReadAllBytes(path), path, ArchiveJson.Default.ArchiveManifest);
public static void WriteArchive(string path, ArchiveManifest manifest)
{
RefuseNewer(path, manifest.SchemaVersion, ArchiveManifest.CurrentSchemaVersion);
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJsonIndented.Default.ArchiveManifest));
}
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
public static SequenceManifest ReadSequence(string path) => ParseSequence(File.ReadAllBytes(path), path);
/// <summary>Parses sidecar bytes (from disk or from the index snapshot).</summary>
public static SequenceManifest ParseSequence(ReadOnlySpan<byte> utf8, string source) => Parse(utf8, source, ArchiveJson.Default.SequenceManifest);
/// <summary>Replaces the sidecar atomically. Compact JSON: a 100k-frame sidecar stays around 35 MB.</summary>
public static void WriteSequence(string path, SequenceManifest manifest)
{
RefuseNewer(path, manifest.SchemaVersion, SequenceManifest.CurrentSchemaVersion);
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
}
/// <summary>
/// Creates the sidecar only if none exists (rename without replace), which makes it the claim on a new
/// sequence folder between processes sharing the archive. False when another sidecar is already there.
/// </summary>
public static bool TryCreateSequence(string path, SequenceManifest manifest)
{
if (File.Exists(path)) return false;
var temp = AtomicFile.TempPathFor(path);
File.WriteAllBytes(temp, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
try
{
using (var fs = new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) fs.Flush(flushToDisk: true);
File.Move(temp, path, overwrite: false);
return true;
}
catch (IOException) when (File.Exists(path))
{
return false;
}
finally
{
if (File.Exists(temp)) File.Delete(temp);
}
}
private static T Parse<T>(ReadOnlySpan<byte> utf8, string source, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo)
{
try
{
return JsonSerializer.Deserialize(utf8, typeInfo) ?? throw new InvalidDataException($"{source} is empty.");
}
catch (JsonException ex)
{
throw new InvalidDataException($"{source} is not a valid Titano file: {ex.Message}", ex);
}
}
private static void RefuseNewer(string path, int version, int supported)
{
if (version > supported) throw new NewerFormatException(path, version);
}
}
+131
View File
@@ -0,0 +1,131 @@
using System.Globalization;
using System.Text.Json;
using Titano.Core.IO;
using Titano.Core.Naming;
namespace Titano.Core.Archiving;
/// <summary>What the planner needs to know about a sequence to rename it.</summary>
public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens);
/// <summary>One sequence's change: its folder, its title, or both.</summary>
public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle)
{
public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal);
}
/// <summary>The exact changes a mass rename will make (its dry run); applying it makes exactly these.</summary>
public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList<RenameStep> Steps);
/// <summary>A plan being applied, with the steps already completed; lets an interrupted rename resume (BRIEF §7).</summary>
public sealed record RenameJournal(RenamePlan Plan, IReadOnlyList<Guid> Completed)
{
public static string PathFor(ArchivePaths archive, Guid planId) => Path.Combine(archive.JournalDirectory, $"rename-{planId:N}.json");
public static void Write(ArchivePaths archive, RenameJournal journal)
{
Directory.CreateDirectory(archive.JournalDirectory);
AtomicFile.WriteAllBytes(PathFor(archive, journal.Plan.Id), JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJsonIndented.Default.RenameJournal));
}
/// <summary>Journals left by interrupted renames.</summary>
public static IReadOnlyList<RenameJournal> ListPending(ArchivePaths archive)
{
if (!Directory.Exists(archive.JournalDirectory)) return [];
var list = new List<RenameJournal>();
foreach (var path in Directory.EnumerateFiles(archive.JournalDirectory, "rename-*.json").Order(StringComparer.Ordinal))
{
if (JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.RenameJournal) is { } journal) list.Add(journal);
}
return list;
}
public static void Delete(ArchivePaths archive, Guid planId) => File.Delete(PathFor(archive, planId));
}
/// <summary>Builds rename plans from folder/title templates (BRIEF §7.1, §12.5).</summary>
public static class RenamePlanner
{
/// <summary>
/// Expands <paramref name="titleTemplate"/> (where <c>{title}</c> is the current title) and then
/// <paramref name="folderTemplate"/> (where <c>{title}</c> is the new title) for every subject. A target folder
/// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"….
/// Null templates leave that part unchanged; subjects without a change produce no step.
/// </summary>
/// <param name="occupied">Every sequence folder of the archive (renamed or not).</param>
public static RenamePlan Plan(Guid archiveId, IReadOnlyList<RenameSubject> subjects, string? folderTemplate, string? titleTemplate,
IEnumerable<string> occupied, Func<string, bool> existsOnDisk)
{
var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate);
var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate);
var claimed = new HashSet<string>(occupied, StringComparer.OrdinalIgnoreCase);
var steps = new List<RenameStep>();
foreach (var s in subjects)
{
var context = new NamingContext
{
CapturedAt = s.FirstCapture ?? DateTime.Today,
SessionStart = s.FirstCapture ?? DateTime.Today,
CameraMake = s.CameraMake,
CameraModel = s.CameraModel,
Lens = s.Lens,
Title = s.Title,
};
var newTitle = title is null ? s.Title : Blank(title.ExpandFile(context)) ?? s.Title;
var newFolder = s.Folder;
// The archive root itself (a folder adopted in place) cannot move; only its title changes.
if (folder is not null && s.Folder.Length > 0 && Blank(folder.ExpandFolder(context with { Title = newTitle })) is { } expanded
&& !expanded.Equals(s.Folder, StringComparison.Ordinal))
{
newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk);
claimed.Add(newFolder);
}
if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle));
}
return new RenamePlan(Guid.CreateVersion7(), archiveId, steps);
}
/// <summary>
/// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted,
/// a case-only change goes through a temporary name (case-insensitive file systems).
/// </summary>
/// <exception cref="IOException">The target is occupied by something else, or neither folder exists.</exception>
public static void MoveFolder(ArchivePaths archive, RenameStep step)
{
if (!step.MovesFolder) return;
var from = archive.FromRelative(step.FromFolder);
var to = archive.FromRelative(step.ToFolder);
var caseOnly = step.FromFolder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase);
if (Directory.Exists(to) && (!caseOnly || HasExactName(to)))
{
if (Directory.Exists(from) && !caseOnly) throw new IOException($"Rename target {step.ToFolder} already exists.");
return;
}
if (!Directory.Exists(from)) throw new DirectoryNotFoundException($"Sequence folder {step.FromFolder} is missing.");
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
if (caseOnly)
{
var temp = from + ".rename-" + Guid.NewGuid().ToString("N")[..8];
Directory.Move(from, temp);
Directory.Move(temp, to);
}
else
{
Directory.Move(from, to);
}
}
private static bool HasExactName(string path) =>
Directory.EnumerateDirectories(Path.GetDirectoryName(path)!, Path.GetFileName(path))
.Any(d => Path.GetFileName(d).Equals(Path.GetFileName(path), StringComparison.Ordinal));
private static string? Blank(string value) => string.IsNullOrWhiteSpace(value) ? null : value;
private static string Unique(string folder, HashSet<string> claimed, Func<string, bool> existsOnDisk)
{
var candidate = folder;
for (var n = 2; claimed.Contains(candidate) || existsOnDisk(candidate); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})");
return candidate;
}
}
@@ -0,0 +1,91 @@
using Titano.Core.Diagnostics;
using Titano.Core.Metadata;
using Titano.Core.Model;
namespace Titano.Core.Archiving;
/// <summary>Files of a sequence folder that the sidecar does not list, and sidecar frames whose file is gone.</summary>
public sealed record SequenceOrphans(string Folder, IReadOnlyList<string> UnindexedFiles, IReadOnlyList<SidecarFrame> MissingFiles)
{
public bool IsEmpty => UnindexedFiles.Count == 0 && MissingFiles.Count == 0;
}
/// <summary>
/// Builds and reconciles sidecars from the photos found in a folder (BRIEF §7, §12.5 "Adopt folder"). Only light
/// metadata is read; originals are never moved, renamed or rewritten. Hashes are left empty for a verify to fill.
/// </summary>
public static class SequenceAdopter
{
/// <summary>A new sidecar for a folder without one. A folder outside the layout (no <c>originals/</c>) is titled by its name.</summary>
public static SequenceManifest Build(Guid id, SequencePaths sequence, string? createdBy, CancellationToken cancellationToken = default)
{
var (frames, first) = ReadFrames(sequence, ArchiveScanner.ListPhotos(sequence), cancellationToken);
var now = DateTimeOffset.UtcNow;
var inLayout = Directory.Exists(sequence.Originals);
var manifest = new SequenceManifest
{
Id = id,
Title = inLayout ? null : Path.GetFileName(Path.TrimEndingDirectorySeparator(sequence.Folder)),
Camera = first is null ? null : new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial },
Lens = first?.LensModel,
CreatedBy = createdBy,
Created = now,
UpdatedAt = now,
};
return SequenceFrames.Insert(manifest, frames);
}
/// <summary>Unindexed photos and missing frame files of one sequence.</summary>
public static SequenceOrphans FindOrphans(SequencePaths sequence, SequenceManifest manifest)
{
var listed = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal);
var unindexed = ArchiveScanner.ListPhotos(sequence).Where(f => !listed.Contains(f)).ToList();
var missing = manifest.Frames.Where(f => !File.Exists(sequence.File(f.File))).ToList();
return new SequenceOrphans(sequence.RelativeFolder, unindexed, missing);
}
/// <summary>
/// Brings a sidecar in line with its folder: unindexed photos are added (in capture order), frames whose file is
/// gone get <see cref="FrameFlags.Missing"/> and frames whose file came back lose it. Null when nothing changed.
/// </summary>
public static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken = default)
{
var orphans = FindOrphans(sequence, manifest);
var missing = orphans.MissingFiles.Select(f => f.Id).ToHashSet();
var flagChanges = new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>>();
foreach (var frame in manifest.Frames)
{
var isMissing = missing.Contains(frame.Id);
if (isMissing == frame.Flags.HasFlag(FrameFlags.Missing)) continue;
flagChanges[frame.Id] = f => f with { Flags = isMissing ? f.Flags | FrameFlags.Missing : f.Flags & ~FrameFlags.Missing };
}
if (flagChanges.Count == 0 && orphans.UnindexedFiles.Count == 0) return null;
var updated = flagChanges.Count == 0 ? manifest : SequenceFrames.Update(manifest, flagChanges);
var (added, first) = ReadFrames(sequence, orphans.UnindexedFiles, cancellationToken);
if (updated.Camera is null && first is not null)
updated = updated with { Camera = new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, Lens = updated.Lens ?? first.LensModel };
return SequenceFrames.Insert(updated, added);
}
private static (List<SidecarFrame> Frames, FrameMetadata? First) ReadFrames(SequencePaths sequence, IReadOnlyList<string> files, CancellationToken cancellationToken)
{
var frames = new List<SidecarFrame>(files.Count);
FrameMetadata? first = null;
foreach (var file in files)
{
cancellationToken.ThrowIfCancellationRequested();
var path = sequence.File(file);
try
{
var metadata = MetadataReader.Read(path, cancellationToken);
if (first is null || (metadata.CapturedAt < first.CapturedAt && metadata.Make is not null)) first = metadata;
frames.Add(SequenceFrames.FromMetadata(Guid.CreateVersion7(), file, Path.GetFileName(file), new FileInfo(path).Length, null, metadata));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Skipped unreadable photo {sequence.ArchiveRelative(file)}", ex);
}
}
return (frames, first);
}
}
@@ -0,0 +1,86 @@
using Titano.Core.Metadata;
using Titano.Core.Model;
namespace Titano.Core.Archiving;
/// <summary>Edits of the frame list of a <see cref="SequenceManifest"/>; frames stay in capture order with gap-free indexes.</summary>
public static class SequenceFrames
{
/// <summary>Capture time (unknown last), then file name.</summary>
public static readonly IComparer<SidecarFrame> CaptureOrder = Comparer<SidecarFrame>.Create((a, b) =>
{
var byTime = (a.CapturedAt ?? DateTime.MaxValue).CompareTo(b.CapturedAt ?? DateTime.MaxValue);
return byTime != 0 ? byTime : string.CompareOrdinal(a.File, b.File);
});
/// <summary>A frame record from light metadata.</summary>
/// <param name="file">Path relative to the sequence folder.</param>
/// <param name="fallbackCapture">Used when the file records no capture time of its own (only a file-system date).</param>
public static SidecarFrame FromMetadata(Guid id, string file, string? original, long size, string? sha256, FrameMetadata metadata,
DateTime? fallbackCapture = null)
{
var useFallback = fallbackCapture is not null && metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown;
return new SidecarFrame
{
Id = id,
File = file,
Original = original,
Size = size,
Sha256 = sha256?.ToLowerInvariant(),
CapturedAt = useFallback ? DateTime.SpecifyKind(fallbackCapture!.Value, DateTimeKind.Unspecified) : metadata.CapturedAt,
UtcOffset = useFallback ? null : metadata.UtcOffsetMinutes,
TimeSource = useFallback ? TimestampSource.Unknown : metadata.TimestampSource,
Exposure = metadata.ExposureTime,
FNumber = metadata.FNumber,
Iso = metadata.Iso,
Focal = metadata.FocalLength,
Bias = metadata.ExposureBias,
Wb = metadata.WhiteBalanceMode,
Orientation = metadata.Orientation,
W = metadata.Width,
H = metadata.Height,
Format = metadata.Format,
PreviewOffset = metadata.LargestPreview?.Offset,
PreviewLength = metadata.LargestPreview?.Length,
};
}
/// <summary>
/// Merges <paramref name="added"/> into the frame list by capture time (existing frames keep their relative order),
/// renumbers indexes and bumps the frames version.
/// </summary>
public static SequenceManifest Insert(SequenceManifest manifest, IReadOnlyList<SidecarFrame> added)
{
if (added.Count == 0) return manifest;
var incoming = added.Order(CaptureOrder).ToList();
var merged = new List<SidecarFrame>(manifest.Frames.Count + incoming.Count);
var next = 0;
foreach (var frame in manifest.Frames)
{
while (next < incoming.Count && CaptureOrder.Compare(incoming[next], frame) < 0) merged.Add(incoming[next++]);
merged.Add(frame);
}
while (next < incoming.Count) merged.Add(incoming[next++]);
return WithFrames(manifest, merged);
}
/// <summary>Replaces the frame list (indexes renumbered) and bumps the frames version.</summary>
public static SequenceManifest WithFrames(SequenceManifest manifest, IReadOnlyList<SidecarFrame> frames)
{
var renumbered = new SidecarFrame[frames.Count];
for (var i = 0; i < frames.Count; i++) renumbered[i] = frames[i].I == i ? frames[i] : frames[i] with { I = i };
return manifest with
{
Frames = renumbered,
Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
};
}
/// <summary>Applies per-frame changes by id (flags, hashes); frames not in <paramref name="changes"/> are kept.</summary>
public static SequenceManifest Update(SequenceManifest manifest, IReadOnlyDictionary<Guid, Func<SidecarFrame, SidecarFrame>> changes)
{
var frames = manifest.Frames.Select(f => changes.TryGetValue(f.Id, out var change) ? change(f) : f).ToList();
return manifest with { Frames = frames, UpdatedAt = DateTimeOffset.UtcNow };
}
}
@@ -0,0 +1,117 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Titano.Core.Imaging;
using Titano.Core.Model;
namespace Titano.Core.Archiving;
/// <summary>
/// <c>.titano/sequence.json</c>, the canonical record of one sequence (BRIEF §7). The database is rebuilt from it,
/// so it carries the ids of the sequence and of every frame. Fields this build does not know are kept in
/// <see cref="Extra"/> and written back unchanged.
/// </summary>
public sealed record SequenceManifest
{
public const int CurrentSchemaVersion = 1;
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
public required Guid Id { get; init; }
public string? Title { get; init; }
public SidecarCamera? Camera { get; init; }
public string? Lens { get; init; }
/// <summary>Frames in capture order; <see cref="SidecarFrame.I"/> equals the position.</summary>
public IReadOnlyList<SidecarFrame> Frames { get; init; } = [];
public IReadOnlyList<string> Tags { get; init; } = [];
public int Rating { get; init; }
public string? Notes { get; init; }
/// <summary><see cref="SequenceStatus"/> bits, stored as a number so bits of newer builds survive.</summary>
public SequenceStatus Status { get; init; }
public ManifestVersions Versions { get; init; } = new();
/// <summary>User id or client name that created the sequence.</summary>
public string? CreatedBy { get; init; }
public DateTimeOffset Created { get; init; }
public DateTimeOffset UpdatedAt { get; init; }
public DateTimeOffset? VerifiedAt { get; init; }
/// <summary>Session keys of automatic grouping (uploads, card imports) that resolve to this sequence.</summary>
public IReadOnlyList<string> SessionKeys { get; init; } = [];
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
public sealed record SidecarCamera
{
public string? Make { get; init; }
public string? Model { get; init; }
public string? Serial { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>Counters the sync protocol compares (BRIEF §14.5).</summary>
public sealed record ManifestVersions
{
public int Settings { get; init; }
public int Frames { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>One frame of <see cref="SequenceManifest"/>; property names follow BRIEF §7.</summary>
public sealed record SidecarFrame
{
public required Guid Id { get; init; }
/// <summary>Index in capture order, from 0.</summary>
public int I { get; init; }
/// <summary>Path relative to the sequence folder, forward slashes (<c>originals/…</c> for imported frames).</summary>
public required string File { get; init; }
/// <summary>Name the file had on the card or upload source.</summary>
public string? Original { get; init; }
public long Size { get; init; }
/// <summary>Lower-case hex SHA-256; null until a verify hashes an adopted file.</summary>
public string? Sha256 { get; init; }
/// <summary>Camera wall-clock time with sub-seconds.</summary>
public DateTime? CapturedAt { get; init; }
public int? UtcOffset { get; init; }
[JsonConverter(typeof(TolerantEnumConverter<TimestampSource>))]
public TimestampSource TimeSource { get; init; }
public double? Exposure { get; init; }
public double? FNumber { get; init; }
public int? Iso { get; init; }
public double? Focal { get; init; }
public double? Bias { get; init; }
public int? Wb { get; init; }
public int Orientation { get; init; } = 1;
public int W { get; init; }
public int H { get; init; }
[JsonConverter(typeof(TolerantEnumConverter<ImageFileFormat>))]
public ImageFileFormat Format { get; init; }
public long? PreviewOffset { get; init; }
public long? PreviewLength { get; init; }
/// <summary><see cref="FrameFlags"/> bits, stored as a number so bits of newer builds survive.</summary>
public FrameFlags Flags { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
+389
View File
@@ -0,0 +1,389 @@
using System.Buffers.Binary;
using Titano.Core.Hashing;
namespace Titano.Core.Archiving;
/// <summary>One thumbnail of a pack: where its JPEG lies and which source it was made from.</summary>
/// <param name="SourceHash">32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale.</param>
public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc);
/// <summary>
/// <c>thumbs.pack</c>: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence
/// instead of thousands. Little-endian blocks after a 16-byte header:
/// <list type="bullet">
/// <item><c>TENT</c> entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes.</item>
/// <item><c>TIDX</c> index: count, one row per frame (latest entry wins), CRC-32 of the rows.</item>
/// <item><c>TFTR</c> footer (last 16 bytes): offset of the index block and its CRC.</item>
/// </list>
/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index
/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall
/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it).
/// </summary>
public static class ThumbPack
{
public const int HashLength = 32;
internal const int HeaderLength = 16;
internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4;
internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4;
internal const int FooterLength = 16;
internal const ushort Version = 1;
/// <summary>Largest thumbnail accepted; anything bigger in a header is corruption.</summary>
internal const int MaxEntryLength = 16 << 20;
internal static ReadOnlySpan<byte> Magic => "TTPK"u8;
internal static ReadOnlySpan<byte> EntryTag => "TENT"u8;
internal static ReadOnlySpan<byte> IndexTag => "TIDX"u8;
internal static ReadOnlySpan<byte> FooterTag => "TFTR"u8;
internal static void WriteHeader(Stream stream)
{
Span<byte> header = stackalloc byte[HeaderLength];
header.Clear();
Magic.CopyTo(header);
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
stream.Write(header);
}
internal static bool HasValidHeader(Stream stream)
{
if (stream.Length < HeaderLength) return false;
Span<byte> header = stackalloc byte[HeaderLength];
stream.Position = 0;
stream.ReadExactly(header);
return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version;
}
/// <summary>
/// The latest entry per frame and where the valid content ends: after the footer when the index is intact
/// (<paramref name="indexOffset"/> = start of that index), otherwise after the last complete block.
/// </summary>
internal static Dictionary<int, ThumbEntry> Load(Stream stream, out long validEnd, out long? indexOffset)
{
indexOffset = null;
if (TryLoadIndex(stream, out var entries, out var offset))
{
validEnd = stream.Length;
indexOffset = offset;
return entries;
}
return Scan(stream, out validEnd);
}
private static bool TryLoadIndex(Stream stream, out Dictionary<int, ThumbEntry> entries, out long indexOffset)
{
entries = [];
indexOffset = 0;
var length = stream.Length;
if (length < HeaderLength + 12 + FooterLength) return false;
Span<byte> footer = stackalloc byte[FooterLength];
stream.Position = length - FooterLength;
stream.ReadExactly(footer);
if (!footer[..4].SequenceEqual(FooterTag)) return false;
indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]);
if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false;
if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false;
Span<byte> head = stackalloc byte[8];
stream.Position = indexOffset;
stream.ReadExactly(head);
var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]);
if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false;
var rows = new byte[count * IndexRowLength + 4];
stream.ReadExactly(rows);
if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false;
for (var i = 0; i < count; i++)
{
var row = rows.AsSpan(i * IndexRowLength, IndexRowLength);
var entry = new ThumbEntry(
BinaryPrimitives.ReadInt32LittleEndian(row),
BinaryPrimitives.ReadInt64LittleEndian(row[4..]),
BinaryPrimitives.ReadInt32LittleEndian(row[12..]),
BinaryPrimitives.ReadUInt16LittleEndian(row[16..]),
BinaryPrimitives.ReadUInt16LittleEndian(row[18..]),
row.Slice(20, HashLength).ToArray(),
BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..]));
if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false;
entries[entry.FrameIndex] = entry;
}
return true;
}
/// <summary>Walks block headers from the start; stops at the first incomplete or unknown block.</summary>
private static Dictionary<int, ThumbEntry> Scan(Stream stream, out long validEnd)
{
var entries = new Dictionary<int, ThumbEntry>();
var length = stream.Length;
var position = (long)HeaderLength;
var header = new byte[EntryHeaderLength];
while (position + 4 <= length)
{
stream.Position = position;
stream.ReadExactly(header.AsSpan(0, 4));
var tag = header.AsSpan(0, 4);
long next;
if (tag.SequenceEqual(EntryTag))
{
if (position + EntryHeaderLength > length) break;
stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4));
var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength));
if (size is < 0 or > MaxEntryLength) break;
next = position + EntryHeaderLength + size;
if (next > length) break;
var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size,
BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)),
header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength)));
}
else if (tag.SequenceEqual(IndexTag))
{
if (position + 8 > length) break;
stream.ReadExactly(header.AsSpan(4, 4));
var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
if (count < 0) break;
next = position + 8 + (long)count * IndexRowLength + 4;
if (next > length) break;
}
else if (tag.SequenceEqual(FooterTag))
{
next = position + FooterLength;
if (next > length) break;
}
else
{
break;
}
position = next;
}
validEnd = position;
return entries;
}
/// <summary>Writes one entry block at the stream position and returns its entry.</summary>
internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
{
var crc = Crc32.Compute(jpeg);
Span<byte> header = stackalloc byte[EntryHeaderLength];
EntryTag.CopyTo(header);
BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex);
BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width);
BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height);
sourceHash.CopyTo(header[12..]);
BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length);
BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc);
var payload = stream.Position + EntryHeaderLength;
stream.Write(header);
stream.Write(jpeg);
return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc);
}
internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection<ThumbEntry> entries)
{
var indexOffset = stream.Position;
var block = new byte[8 + entries.Count * IndexRowLength + 4];
IndexTag.CopyTo(block);
BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count);
var i = 0;
foreach (var e in entries.OrderBy(e => e.FrameIndex))
{
var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength);
BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex);
BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset);
BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length);
BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width);
BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height);
e.SourceHash.CopyTo(row[20..]);
BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc);
}
var rowsLength = entries.Count * IndexRowLength;
BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength)));
stream.Write(block);
Span<byte> footer = stackalloc byte[FooterLength];
FooterTag.CopyTo(footer);
BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset);
BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12]));
stream.Write(footer);
}
/// <summary>Size of a pack holding only <paramref name="entries"/>; the rest of a file is reclaimable garbage.</summary>
public static long LiveBytes(IEnumerable<ThumbEntry> entries)
{
long count = 0, payload = 0;
foreach (var e in entries)
{
count++;
payload += EntryHeaderLength + e.Length;
}
return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength;
}
/// <summary>A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known.</summary>
public static byte[] Fingerprint(string? sha256, string file, long size) =>
sha256 is { Length: HashLength * 2 }
? Convert.FromHexString(sha256)
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
}
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
public sealed class ThumbPackReader : IDisposable
{
private readonly FileStream? _stream;
private readonly Lock _gate = new();
private ThumbPackReader(FileStream? stream, Dictionary<int, ThumbEntry> entries)
{
_stream = stream;
Entries = entries;
}
/// <summary>The latest entry of every frame that has one.</summary>
public IReadOnlyDictionary<int, ThumbEntry> Entries { get; }
/// <summary>Bytes held by replaced entries (reclaimed by <see cref="ThumbPackWriter.Compact"/>).</summary>
public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values));
public long Length => _stream?.Length ?? 0;
/// <summary>Opens a pack; a missing or damaged pack reads as empty.</summary>
public static ThumbPackReader Open(string path)
{
if (!File.Exists(path)) return new ThumbPackReader(null, []);
var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess);
try
{
var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : [];
return new ThumbPackReader(stream, entries);
}
catch
{
stream.Dispose();
throw;
}
}
/// <summary>The JPEG of a frame, or null when absent or damaged (CRC mismatch).</summary>
public byte[]? Read(int frameIndex)
{
if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null;
var data = new byte[entry.Length];
lock (_gate)
{
if (entry.Offset + entry.Length > _stream.Length) return null;
_stream.Position = entry.Offset;
_stream.ReadExactly(data);
}
return Crc32.Compute(data) == entry.Crc ? data : null;
}
public void Dispose() => _stream?.Dispose();
}
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it).</summary>
public sealed class ThumbPackWriter : IDisposable
{
private readonly FileStream _stream;
private readonly Dictionary<int, ThumbEntry> _entries;
private long? _indexOffset;
private bool _dirty;
private ThumbPackWriter(FileStream stream, Dictionary<int, ThumbEntry> entries, long? indexOffset)
{
_stream = stream;
_entries = entries;
_indexOffset = indexOffset;
}
public IReadOnlyDictionary<int, ThumbEntry> Entries => _entries;
/// <summary>Opens or creates a pack, dropping an incomplete tail left by a crash.</summary>
public static ThumbPackWriter Open(string path)
{
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16);
try
{
if (!ThumbPack.HasValidHeader(stream))
{
// Not a pack (or a header torn by a crash): thumbnails are regenerable, start over.
stream.SetLength(0);
ThumbPack.WriteHeader(stream);
return new ThumbPackWriter(stream, [], null);
}
var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset);
if (validEnd < stream.Length) stream.SetLength(validEnd);
return new ThumbPackWriter(stream, entries, indexOffset);
}
catch
{
stream.Dispose();
throw;
}
}
/// <summary>Appends a thumbnail; it replaces any earlier entry of the frame once committed.</summary>
public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
{
ArgumentOutOfRangeException.ThrowIfNegative(frameIndex);
if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash));
if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg));
if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width));
// The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk).
_stream.Position = _indexOffset ?? _stream.Length;
if (_indexOffset is { } offset) _stream.SetLength(offset);
_indexOffset = null;
_entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg);
_dirty = true;
}
/// <summary>Writes the index and footer and flushes to disk.</summary>
public void Commit()
{
if (!_dirty) return;
_stream.Position = _stream.Length;
var indexOffset = _stream.Position;
ThumbPack.WriteIndexAndFooter(_stream, _entries.Values);
_stream.Flush(flushToDisk: true);
_indexOffset = indexOffset;
_dirty = false;
}
/// <summary>Rewrites the pack with only its live entries (atomically, so readers keep a consistent file).</summary>
public static void Compact(string path)
{
var temp = IO.AtomicFile.TempPathFor(path);
try
{
using (var source = ThumbPackReader.Open(path))
using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
{
ThumbPack.WriteHeader(target);
var entries = new List<ThumbEntry>(source.Entries.Count);
foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex))
{
if (source.Read(entry.FrameIndex) is { } jpeg)
entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg));
}
ThumbPack.WriteIndexAndFooter(target, entries);
}
IO.AtomicFile.Commit(temp, path);
}
finally
{
if (File.Exists(temp)) File.Delete(temp);
}
}
public void Dispose()
{
try
{
Commit();
}
finally
{
_stream.Dispose();
}
}
}
@@ -0,0 +1,182 @@
using Titano.Core.Archiving;
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Imaging;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Testing;
namespace Titano.Core.Tests.Archiving;
[Category("archive")]
public sealed class ArchiveFolderTests(TestContext context)
{
private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
private ArchivePaths NewArchive() => new(context.CreateTempDirectory("archive"));
[Test]
public void SequenceFoldersAreFoundInLayoutAndAdoptedForms()
{
var archive = NewArchive();
TestPhotos.WriteSeries(Path.Combine(archive.Root, "2026", "2026-09-28 Canon EOS R5", "originals"), Start, 3, TimeSpan.FromSeconds(10));
TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour"), Start, 2, TimeSpan.FromSeconds(10));
TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour", "selects"), Start, 1, TimeSpan.FromSeconds(10));
TestPhotos.WriteSeries(Path.Combine(archive.Root, ".hidden"), Start, 1, TimeSpan.FromSeconds(10));
File.WriteAllBytes(Path.Combine(archive.Root, "Trips", "._IMG_0001.JPG"), [1]);
Directory.CreateDirectory(Path.Combine(archive.Root, "empty"));
var folders = ArchiveScanner.FindSequenceFolders(archive);
Assert.SequenceEqual(["2026/2026-09-28 Canon EOS R5", "Trips/Harbour", "Trips/Harbour/selects"], folders.Select(f => f.RelativeFolder));
Assert.True(folders.All(f => f.Sidecar is null && f.Photos is not null));
Assert.SequenceEqual(["originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG"],
ArchiveScanner.ListPhotos(archive.Sequence("2026/2026-09-28 Canon EOS R5")));
}
[Test]
public void AdoptionBuildsASidecarInCaptureOrderWithoutTouchingPhotos()
{
var archive = NewArchive();
var folder = Path.Combine(archive.Root, "Harbour sunset");
// File names run against capture order: the sidecar follows capture time.
TestPhotos.Write(Path.Combine(folder, "a.jpg"), Start.AddSeconds(20), 1);
TestPhotos.Write(Path.Combine(folder, "b.jpg"), Start, 2);
TestPhotos.Write(Path.Combine(folder, "c.jpg"), Start.AddSeconds(10), 3);
var before = Directory.GetFiles(folder).ToDictionary(f => f, File.ReadAllBytes);
var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), archive.Sequence("Harbour sunset"), "test");
Assert.SequenceEqual(["b.jpg", "c.jpg", "a.jpg"], manifest.Frames.Select(f => f.File));
Assert.SequenceEqual([0, 1, 2], manifest.Frames.Select(f => f.I));
Assert.Equal("Harbour sunset", manifest.Title);
Assert.Equal("Canon EOS R5", manifest.Camera!.Model);
Assert.True(manifest.Frames.All(f => f.Sha256 is null && f.Size > 0 && f.Format == ImageFileFormat.Jpeg && f.W == 48));
foreach (var (path, bytes) in before) Assert.BytesEqual(bytes, File.ReadAllBytes(path));
}
[Test]
public void ReconcileIndexesNewPhotosAndFlagsMissingOnes()
{
var archive = NewArchive();
var sequence = archive.Sequence("s");
var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10));
var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null);
Assert.Null(SequenceAdopter.Reconcile(sequence, manifest));
File.Delete(files[1]);
TestPhotos.Write(Path.Combine(sequence.Originals, "IMG_0000.JPG"), Start.AddSeconds(-10), 9);
var orphans = SequenceAdopter.FindOrphans(sequence, manifest);
Assert.SequenceEqual(["originals/IMG_0000.JPG"], orphans.UnindexedFiles);
Assert.SequenceEqual(["originals/IMG_0002.JPG"], orphans.MissingFiles.Select(f => f.File));
var reconciled = SequenceAdopter.Reconcile(sequence, manifest)!;
Assert.SequenceEqual(["originals/IMG_0000.JPG", "originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG", "originals/IMG_0004.JPG"],
reconciled.Frames.Select(f => f.File));
Assert.Equal(FrameFlags.Missing, reconciled.Frames[2].Flags);
Assert.Equal(manifest.Frames[0].Id, reconciled.Frames[1].Id);
TestPhotos.Write(files[1], Start.AddSeconds(10), 2);
var restored = SequenceAdopter.Reconcile(sequence, reconciled)!;
Assert.Equal(FrameFlags.None, restored.Frames[2].Flags);
}
[Test]
public async Task VerifyDetectsAFlippedByteAMissingFileAndAChangedFile()
{
var archive = NewArchive();
var sequence = archive.Sequence("s");
var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10));
var adopted = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null);
var hashed = new List<SidecarFrame>();
foreach (var frame in adopted.Frames)
{
var check = await FrameVerifier.CheckAsync(sequence, frame);
Assert.Equal(FrameHealth.Hashed, check.Health);
hashed.Add(check.Updated);
}
var bytes = File.ReadAllBytes(files[0]);
bytes[bytes.Length / 2] ^= 0x01;
File.WriteAllBytes(files[0], bytes);
File.Delete(files[1]);
File.AppendAllText(files[2], "edit");
var health = new List<FrameHealth>();
foreach (var frame in hashed) health.Add((await FrameVerifier.CheckAsync(sequence, frame)).Health);
Assert.SequenceEqual([FrameHealth.Corrupt, FrameHealth.Missing, FrameHealth.Changed, FrameHealth.Ok], health);
var corrupt = await FrameVerifier.CheckAsync(sequence, hashed[0]);
Assert.Equal(FrameFlags.Corrupt, corrupt.Flags);
Assert.Equal(hashed[0].Sha256, corrupt.Updated.Sha256);
}
[Test]
public void TheIndexSnapshotIsUsedOnlyForUnchangedSidecars()
{
var archive = NewArchive();
Directory.CreateDirectory(archive.MetaDirectory);
var stampA = new FileStamp(10, new DateTime(2026, 9, 1, 0, 0, 0, DateTimeKind.Utc));
using (var writer = new IndexSnapshot.Writer(archive.SnapshotPath))
{
writer.Add("a", stampA, "{\"a\":1}"u8);
writer.Add("b/ç", new FileStamp(20, stampA.ModifiedUtc), "{\"b\":2}"u8);
writer.Commit();
}
var snapshot = IndexSnapshot.TryLoad(archive.SnapshotPath)!;
Assert.Equal(2, snapshot.Count);
Assert.BytesEqual("{\"a\":1}"u8, snapshot.TryGet("a", stampA));
Assert.Null(snapshot.TryGet("a", stampA with { Size = 11 }));
Assert.NotNull(snapshot.TryGet("b/ç", new FileStamp(20, stampA.ModifiedUtc)));
var bytes = File.ReadAllBytes(archive.SnapshotPath);
bytes[12] ^= 0x40;
File.WriteAllBytes(archive.SnapshotPath, bytes);
Assert.Null(IndexSnapshot.TryLoad(archive.SnapshotPath));
}
[Test]
public void RenamePlansAvoidTakenFoldersAndMoveIdempotently()
{
var archive = NewArchive();
Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old a"));
Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old b"));
Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "2026-09-28 Sunset"));
var a = new RenameSubject(Guid.NewGuid(), "2026/old a", null, Start, "Canon", "Canon EOS R5", null);
var b = new RenameSubject(Guid.NewGuid(), "2026/old b", "Dawn", Start, "Canon", "Canon EOS R5", null);
var plan = RenamePlanner.Plan(Guid.NewGuid(), [a, b], "{year}/{date} {title}", "Sunset",
["2026/old a", "2026/old b"], f => Directory.Exists(archive.FromRelative(f)));
Assert.SequenceEqual(["2026/2026-09-28 Sunset (2)", "2026/2026-09-28 Sunset (3)"], plan.Steps.Select(s => s.ToFolder));
Assert.True(plan.Steps.All(s => s.ToTitle == "Sunset"));
Assert.Equal("Dawn", plan.Steps[1].FromTitle);
RenamePlanner.MoveFolder(archive, plan.Steps[0]);
RenamePlanner.MoveFolder(archive, plan.Steps[0]);
Assert.False(Directory.Exists(archive.FromRelative("2026/old a")));
Assert.True(Directory.Exists(archive.FromRelative("2026/2026-09-28 Sunset (2)")));
var titleOnly = RenamePlanner.Plan(Guid.NewGuid(), [b], null, "{title} morning", ["2026/old b"], _ => false);
Assert.Equal("2026/old b", titleOnly.Steps.Single().ToFolder);
Assert.Equal("Dawn morning", titleOnly.Steps.Single().ToTitle);
Assert.Empty(RenamePlanner.Plan(Guid.NewGuid(), [b], "2026/old b", null, ["2026/old b"], _ => true).Steps);
}
[Test]
public void ThumbnailsAndProxiesAreRenderedFromTheFrame()
{
var archive = NewArchive();
var sequence = archive.Sequence("s");
Directory.CreateDirectory(sequence.Originals);
File.WriteAllBytes(Path.Combine(sequence.Originals, "big.jpg"), TestPhotos.Jpeg(Start, 7, width: 640, height: 480, orientation: 6));
var frame = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null).Frames.Single();
var renderer = new FrameImageRenderer(new ImageDecoder([]));
var thumb = renderer.Thumbnail(sequence, frame)!;
Assert.Equal((240, 320), (thumb.Width, thumb.Height));
Assert.Equal(DecodeTier.Native, thumb.Tier);
Assert.Equal(240, JpegDecoder.ReadInfo(thumb.Jpeg).Width);
var proxy = renderer.Proxy(sequence, frame, new SidecarCamera { Make = "Canon", Model = "Canon EOS R5" }, new ProxyPolicy { LongEdge = 400 })!;
Assert.Equal((300, 400), (proxy.Width, proxy.Height));
var exif = MetadataReader.Read(new MemoryStream(proxy.Jpeg), "p.jpg");
Assert.Equal(Start, exif.CapturedAt);
Assert.Equal("Canon EOS R5", exif.Model);
Assert.Equal(1, exif.Orientation);
}
}
@@ -0,0 +1,113 @@
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Testing;
namespace Titano.Core.Tests.Archiving;
[Category("archive")]
public sealed class LeaseFileTests(TestContext context)
{
private static void WriteRecord(string path, LeaseRecord record) =>
File.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
[Test]
public void ALeaseIsExclusiveUntilReleased()
{
var locks = context.CreateTempDirectory();
using (var a = LeaseFile.TryAcquire(locks, "seq-1", "server"))
{
Assert.NotNull(a);
Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client"));
using var other = LeaseFile.TryAcquire(locks, "seq-2", "client");
Assert.NotNull(other);
}
using var b = LeaseFile.TryAcquire(locks, "seq-1", "client");
Assert.NotNull(b);
Assert.Equal("client", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner);
}
[Test]
public void AnExpiredLeaseOfACrashedHolderIsTakenOver()
{
var locks = context.CreateTempDirectory();
var past = DateTimeOffset.UtcNow.AddMinutes(-10);
WriteRecord(LeaseFile.PathFor(locks, "seq-1"), new LeaseRecord("dead", "crashed", "pc", 1, past, past.AddMinutes(2)));
using var lease = LeaseFile.TryAcquire(locks, "seq-1", "survivor");
Assert.NotNull(lease);
Assert.Equal("survivor", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner);
Assert.Count(1, Directory.GetFiles(locks));
}
[Test]
public async Task AHeldLeaseIsRenewedPastItsDuration()
{
var locks = context.CreateTempDirectory();
using var lease = LeaseFile.TryAcquire(locks, "seq-1", "server", TimeSpan.FromMilliseconds(300));
Assert.NotNull(lease);
await Task.Delay(900);
Assert.True(lease.IsHeld);
Assert.Greater((LeaseFile.Read(lease.Path)!.Expires - DateTimeOffset.UtcNow).TotalMilliseconds, 0);
Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client", TimeSpan.FromMilliseconds(300)));
}
[Test]
public void AFreshEmptyLeaseFileIsNotStaleButAnOldOneIs()
{
var locks = context.CreateTempDirectory();
var path = LeaseFile.PathFor(locks, "seq-1");
File.WriteAllBytes(path, []);
Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client"));
File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddHours(-1));
using var lease = LeaseFile.TryAcquire(locks, "seq-1", "client");
Assert.NotNull(lease);
}
[Test]
public void ALostLeaseIsDetectedAndNotDeletedByItsFormerHolder()
{
var locks = context.CreateTempDirectory();
var lease = LeaseFile.TryAcquire(locks, "seq-1", "slow");
Assert.NotNull(lease);
var now = DateTimeOffset.UtcNow;
WriteRecord(lease.Path, new LeaseRecord("thief", "other", "pc", 2, now, now.AddMinutes(2)));
Assert.False(lease.Renew());
Assert.False(lease.IsHeld);
lease.Dispose();
Assert.Equal("thief", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Token);
}
[Test]
public async Task WaitingForABusyLeaseTimesOutWithItsHolder()
{
var locks = context.CreateTempDirectory();
using var held = LeaseFile.TryAcquire(locks, "seq-1", "server");
var error = await Assert.ThrowsAsync<LeaseUnavailableException>(() =>
LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromMilliseconds(200)));
Assert.Equal("server", error.Holder!.Owner);
}
[Test]
public async Task WaitingForALeaseSucceedsOnceItIsReleased()
{
var locks = context.CreateTempDirectory();
var held = LeaseFile.TryAcquire(locks, "seq-1", "server")!;
var waiting = LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromSeconds(10));
await Task.Delay(100);
held.Dispose();
using var lease = await waiting;
Assert.Equal("client", lease.Record.Owner);
}
[Test]
public void CleanupRemovesOnlyStaleLeasesAndItsDryRunNothing()
{
var locks = context.CreateTempDirectory();
var past = DateTimeOffset.UtcNow.AddMinutes(-10);
WriteRecord(LeaseFile.PathFor(locks, "dead"), new LeaseRecord("t", "crashed", "pc", 1, past, past.AddMinutes(1)));
using var live = LeaseFile.TryAcquire(locks, "live", "server");
Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: true));
Assert.Count(2, Directory.GetFiles(locks));
Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: false));
Assert.SequenceEqual([LeaseFile.PathFor(locks, "live")], Directory.GetFiles(locks));
}
}
@@ -0,0 +1,157 @@
using System.Text;
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Imaging;
using Titano.Core.IO;
using Titano.Core.Model;
using Titano.Testing;
namespace Titano.Core.Tests.Archiving;
[Category("archive")]
public sealed class ManifestStoreTests(TestContext context)
{
private static SequenceManifest Sample() => new()
{
Id = Guid.CreateVersion7(),
Title = "Harbour sunset",
Camera = new SidecarCamera { Make = "Canon", Model = "Canon EOS R5", Serial = "0431" },
Lens = "RF24-70mm F2.8 L IS USM",
Frames =
[
new SidecarFrame
{
Id = Guid.CreateVersion7(), I = 0, File = "originals/2026-09-28_18-02-11_0001.CR2", Original = "IMG_0001.CR2", Size = 25_000_000,
Sha256 = new string('a', 64), CapturedAt = new DateTime(2026, 9, 28, 18, 2, 11, 250), UtcOffset = 120,
TimeSource = TimestampSource.ExifOriginal, Exposure = 1.0 / 250, FNumber = 8, Iso = 100, Focal = 24, Bias = -0.3, Wb = 0,
Orientation = 6, W = 8192, H = 5464, Format = ImageFileFormat.Cr2, PreviewOffset = 1234, PreviewLength = 567_890,
Flags = FrameFlags.Excluded,
},
],
Tags = ["sunset", "harbour"],
Rating = 4,
Notes = "Windy",
Status = SequenceStatus.ThumbsBuilt | SequenceStatus.Verified,
Versions = new ManifestVersions { Settings = 3, Frames = 7 },
CreatedBy = "alberto",
Created = new DateTimeOffset(2026, 9, 28, 20, 0, 0, TimeSpan.Zero),
UpdatedAt = new DateTimeOffset(2026, 9, 29, 8, 0, 0, TimeSpan.Zero),
VerifiedAt = new DateTimeOffset(2026, 9, 30, 8, 0, 0, TimeSpan.Zero),
SessionKeys = ["card-1234abcd-20260928-180211"],
};
private string SidecarPath() => Path.Combine(context.CreateTempDirectory(), ".titano", "sequence.json");
private static string Canonical(SequenceManifest m) => JsonSerializer.Serialize(m, ArchiveJson.Default.SequenceManifest);
[Test]
public void SequenceSidecarRoundTripsEveryField()
{
var path = SidecarPath();
var original = Sample();
ManifestStore.WriteSequence(path, original);
var read = ManifestStore.ReadSequence(path);
Assert.Equal(Canonical(original), Canonical(read));
Assert.Equal(original.Frames[0], read.Frames[0] with { Extra = original.Frames[0].Extra });
var json = File.ReadAllText(path);
foreach (var field in new[] { "\"i\":", "\"file\":", "\"sha256\":", "\"capturedAt\":", "\"fNumber\":", "\"w\":", "\"previewOffset\":", "\"flags\":" })
Assert.Contains(field, json);
}
[Test]
public void UnknownFieldsAndEnumValuesFromANewerBuildAreReadAndKept()
{
var path = SidecarPath();
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var id = Guid.CreateVersion7();
var frameId = Guid.CreateVersion7();
File.WriteAllText(path, $$"""
{
"schemaVersion": 1, "id": "{{id}}", "title": "Comet", "futureBlock": { "a": [1, 2] },
"camera": { "make": "Nikon", "sensorTemp": 31 },
"frames": [ { "id": "{{frameId}}", "i": 0, "file": "x.nef", "size": 10, "format": "Nef2030", "timeSource": "Atomic",
"flags": 4096, "aiScore": 0.9 } ],
"status": 1024, "versions": { "settings": 1, "frames": 2, "renders": 5 },
}
""");
var read = ManifestStore.ReadSequence(path);
Assert.Equal(id, read.Id);
Assert.Equal(ImageFileFormat.Unknown, read.Frames[0].Format);
Assert.Equal(TimestampSource.Unknown, read.Frames[0].TimeSource);
Assert.Equal((FrameFlags)4096, read.Frames[0].Flags);
ManifestStore.WriteSequence(path, read with { Title = "Comet 2026" });
var json = File.ReadAllText(path);
foreach (var kept in new[] { "futureBlock", "sensorTemp", "aiScore", "renders", "4096", "1024" }) Assert.Contains(kept, json);
Assert.Equal("Comet 2026", ManifestStore.ReadSequence(path).Title);
}
[Test]
public void ANewerSchemaIsReadableButNeverRewritten()
{
var path = SidecarPath();
ManifestStore.WriteSequence(path, Sample());
var newer = ManifestStore.ReadSequence(path) with { SchemaVersion = SequenceManifest.CurrentSchemaVersion + 1 };
var before = File.ReadAllBytes(path);
Assert.Throws<NewerFormatException>(() => ManifestStore.WriteSequence(path, newer));
Assert.BytesEqual(before, File.ReadAllBytes(path));
}
[Test]
public void DamagedJsonIsReportedAsInvalidData()
{
var path = SidecarPath();
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
File.WriteAllText(path, "{ \"id\": ");
Assert.Throws<InvalidDataException>(() => ManifestStore.ReadSequence(path));
File.WriteAllText(path, "{ \"title\": \"no id\" }");
Assert.Throws<InvalidDataException>(() => ManifestStore.ReadSequence(path));
}
[Test]
public void ACrashDuringAWriteLeavesThePreviousSidecarIntact()
{
var path = SidecarPath();
var first = Sample();
ManifestStore.WriteSequence(path, first);
// Simulated crash: the next version was half written to the temporary sibling and never renamed.
var torn = AtomicFile.TempPathFor(path);
File.WriteAllBytes(torn, Encoding.UTF8.GetBytes(Canonical(first with { Title = "Lost" })[..40]));
Assert.Equal(Canonical(first), Canonical(ManifestStore.ReadSequence(path)));
ManifestStore.WriteSequence(path, first with { Title = "Second" });
Assert.Equal("Second", ManifestStore.ReadSequence(path).Title);
Assert.FileExists(torn);
Assert.Equal(2, Directory.GetFiles(Path.GetDirectoryName(path)!).Length);
}
[Test]
public void CreatingASidecarClaimsTheFolderOnce()
{
var path = SidecarPath();
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var a = Sample();
var b = Sample();
Assert.True(ManifestStore.TryCreateSequence(path, a));
Assert.False(ManifestStore.TryCreateSequence(path, b));
Assert.Equal(a.Id, ManifestStore.ReadSequence(path).Id);
Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(path)!));
}
[Test]
public void ArchiveManifestRoundTripsAndKeepsUnknownFields()
{
var path = Path.Combine(context.CreateTempDirectory(), "archive.json");
File.WriteAllText(path, $$"""
{ "schemaVersion": 1, "id": "{{Guid.CreateVersion7()}}", "name": "Main", "created": "2026-10-02T09:00:00+00:00",
"layoutTemplates": { "folder": "{year}/{title}", "file": "{n:00000}" }, "proxyPolicy": { "longEdge": 2560, "quality": 90 },
"syncPolicy": "mirror" }
""");
var read = ManifestStore.ReadArchive(path);
Assert.Equal("{year}/{title}", read.LayoutTemplates.Folder);
Assert.Equal(2560, read.ProxyPolicy.LongEdge);
ManifestStore.WriteArchive(path, read with { Name = "Studio" });
Assert.Contains("syncPolicy", File.ReadAllText(path));
Assert.Equal("Studio", ManifestStore.ReadArchive(path).Name);
}
}
@@ -0,0 +1,51 @@
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Imaging;
using Titano.Core.Metadata;
namespace Titano.Core.Tests.Archiving;
/// <summary>Small real JPEGs with Exif (capture time, camera) made by the in-house encoder; each seed gives distinct bytes.</summary>
internal static class TestPhotos
{
public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431",
int width = 48, int height = 32, int orientation = 1)
{
using var image = new ImageBuffer(width, height);
var random = new Random(seed);
for (var c = 0; c < 3; c++)
{
var plane = image.Plane(c);
var level = (float)random.NextDouble();
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % width) / width);
}
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata
{
Make = make,
Model = model,
BodySerial = serial,
CapturedAt = captured,
ExposureTime = 1.0 / 125,
FNumber = 8,
Iso = 100,
Orientation = orientation,
});
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
}
public static string Write(string path, DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431")
{
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
File.WriteAllBytes(path, Jpeg(captured, seed, make, model, serial));
return path;
}
/// <summary>Writes <paramref name="count"/> photos named IMG_0001.JPG… taken every <paramref name="interval"/>.</summary>
public static List<string> WriteSeries(string directory, DateTime start, int count, TimeSpan interval, int firstSeed = 1, string prefix = "IMG_",
string make = "Canon", string model = "Canon EOS R5", int firstNumber = 1)
{
var paths = new List<string>(count);
for (var i = 0; i < count; i++)
paths.Add(Write(Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG"), start + interval * i, firstSeed + i, make, model));
return paths;
}
}
@@ -0,0 +1,165 @@
using Titano.Core.Archiving;
using Titano.Testing;
namespace Titano.Core.Tests.Archiving;
[Category("archive")]
public sealed class ThumbPackTests(TestContext context)
{
private static byte[] Payload(int seed, int length = 300)
{
var data = new byte[length];
new Random(seed).NextBytes(data);
return data;
}
private static byte[] Hash(int seed) => ThumbPack.Fingerprint(null, $"originals/{seed}.jpg", seed);
private string NewPack() => Path.Combine(context.CreateTempDirectory(), ".titano", "thumbs.pack");
private static void Fill(string path, int count, int seedOffset = 0)
{
using var writer = ThumbPackWriter.Open(path);
for (var i = 0; i < count; i++) writer.Append(i, 320, 213, Hash(i + seedOffset), Payload(i + seedOffset));
}
[Test]
public void EntriesAreReadBackByFrameIndex()
{
var path = NewPack();
Fill(path, 100);
using var reader = ThumbPackReader.Open(path);
Assert.Equal(100, reader.Entries.Count);
Assert.BytesEqual(Payload(57), reader.Read(57));
Assert.BytesEqual(Hash(57), reader.Entries[57].SourceHash);
Assert.Equal(320, reader.Entries[57].Width);
Assert.Null(reader.Read(100));
}
[Test]
public void AppendingAgainReplacesAFrameAndKeepsTheOthers()
{
var path = NewPack();
Fill(path, 3);
using (var writer = ThumbPackWriter.Open(path))
{
Assert.Equal(3, writer.Entries.Count);
writer.Append(1, 200, 100, Hash(99), Payload(99));
writer.Append(3, 200, 100, Hash(3), Payload(3));
}
using var reader = ThumbPackReader.Open(path);
Assert.Equal(4, reader.Entries.Count);
Assert.BytesEqual(Payload(99), reader.Read(1));
Assert.BytesEqual(Payload(0), reader.Read(0));
Assert.BytesEqual(Payload(3), reader.Read(3));
}
[Test]
public void AnIncompleteTailIsIgnoredAndTruncatedByTheNextWriter()
{
var path = NewPack();
Fill(path, 2);
var committed = new FileInfo(path).Length;
// Simulated crash mid-append: an entry header and part of its payload, no index.
using (var stream = new FileStream(path, FileMode.Append))
{
stream.Write("TENT"u8);
stream.Write(BitConverter.GetBytes(2));
stream.Write(new byte[30]);
}
using (var reader = ThumbPackReader.Open(path))
{
Assert.Equal(2, reader.Entries.Count);
Assert.BytesEqual(Payload(1), reader.Read(1));
}
using (var writer = ThumbPackWriter.Open(path))
{
Assert.Equal(committed, new FileInfo(path).Length);
writer.Append(2, 320, 213, Hash(2), Payload(2));
}
using var after = ThumbPackReader.Open(path);
Assert.Equal(3, after.Entries.Count);
Assert.BytesEqual(Payload(2), after.Read(2));
}
[Test]
public void EntriesWrittenAfterTheLastIndexAreFoundWithoutIt()
{
var path = NewPack();
Fill(path, 5);
// Simulated crash after new entries overwrote the old index but before the new index was written.
var withoutIndex = File.ReadAllBytes(path);
using (var reader = ThumbPackReader.Open(path))
{
var end = reader.Entries.Values.Max(e => e.Offset + e.Length);
File.WriteAllBytes(path, withoutIndex[..(int)end]);
}
using var scanned = ThumbPackReader.Open(path);
Assert.Equal(5, scanned.Entries.Count);
for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), scanned.Read(i));
}
[Test]
public void ADamagedThumbnailReadsAsMissing()
{
var path = NewPack();
Fill(path, 3);
long offset;
using (var reader = ThumbPackReader.Open(path)) offset = reader.Entries[1].Offset;
using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite))
{
stream.Position = offset + 10;
var b = stream.ReadByte();
stream.Position = offset + 10;
stream.WriteByte((byte)(b ^ 0xFF));
}
using var damaged = ThumbPackReader.Open(path);
Assert.Null(damaged.Read(1));
Assert.BytesEqual(Payload(2), damaged.Read(2));
}
[Test]
public void AnOpenReaderKeepsReadingWhileAWriterAppends()
{
var path = NewPack();
Fill(path, 3);
using var reader = ThumbPackReader.Open(path);
using (var writer = ThumbPackWriter.Open(path))
{
for (var i = 3; i < 50; i++) writer.Append(i, 320, 213, Hash(i), Payload(i, 2000));
}
Assert.BytesEqual(Payload(0), reader.Read(0));
Assert.BytesEqual(Payload(2), reader.Read(2));
using var fresh = ThumbPackReader.Open(path);
Assert.Equal(50, fresh.Entries.Count);
}
[Test]
public void CompactionDropsReplacedEntries()
{
var path = NewPack();
for (var round = 0; round < 4; round++) Fill(path, 20, seedOffset: round * 100);
long garbage;
using (var reader = ThumbPackReader.Open(path)) garbage = reader.GarbageBytes;
Assert.Greater(garbage, 0);
var before = new FileInfo(path).Length;
ThumbPackWriter.Compact(path);
Assert.Equal(before - garbage, new FileInfo(path).Length);
using var compacted = ThumbPackReader.Open(path);
Assert.Equal(0L, compacted.GarbageBytes);
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
}
[Test]
public void AMissingOrForeignFileReadsAsEmpty()
{
var path = NewPack();
using (var none = ThumbPackReader.Open(path)) Assert.Equal(0, none.Entries.Count);
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
File.WriteAllText(path, "not a pack");
using (var foreign = ThumbPackReader.Open(path)) Assert.Equal(0, foreign.Entries.Count);
Fill(path, 1);
using var rewritten = ThumbPackReader.Open(path);
Assert.Equal(1, rewritten.Entries.Count);
}
}