feat(core): archive layout, sidecars, leases and thumbnail packs
archive.json and sequence.json with source-generated JSON, forward-compatible reading (unknown fields kept, unknown enum names tolerated) and atomic writes; lease files with expiry and stale takeover; thumbs.pack with index, crash-safe append and random access; folder scanning, in-place adoption, reconcile, verify, index snapshot, rename planning and thumbnail/proxy rendering. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
24 files changed
+2556
No files matched your search
@@ -5,4 +5,26 @@
|
||||
<resheader name="reader"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
|
||||
<resheader name="writer"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
|
||||
<data name="Archive.Empty" xml:space="preserve"><value>No archive yet</value></data>
|
||||
<data name="Archive.Error.Busy" xml:space="preserve"><value>Another Titano ({0}) is writing here. Try again shortly.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.CopyMismatch" xml:space="preserve"><value>The copy of {0} does not match the original.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.DuplicateLost" xml:space="preserve"><value>{0} could not be added to the sequence.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.NewerFormat" xml:space="preserve"><value>{0} was written by a newer Titano (format {1}).</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.Offline" xml:space="preserve"><value>The archive {0} is offline.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.ReadOnly" xml:space="preserve"><value>The archive {0} is read-only.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.SourceChanged" xml:space="preserve"><value>{0} changed after the plan was made.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.SourceGone" xml:space="preserve"><value>The card or folder {0} is no longer available.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.StalePlan" xml:space="preserve"><value>The archive changed after this plan was made.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Log.DryRun" xml:space="preserve"><value>Dry run: nothing was changed.</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.FolderFailed" xml:space="preserve"><value>Not indexed: {0} ({1})</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.FrameChanged" xml:space="preserve"><value>Changed: {0}</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.FrameCorrupt" xml:space="preserve"><value>Damaged: {0}</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.FrameMissing" xml:space="preserve"><value>Missing: {0}</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.ImagesDone" xml:space="preserve"><value>{0}: {1} made, {2} reused, {3} unreadable</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.ImportDone" xml:space="preserve"><value>{0} copied, {1} skipped, {2} failed</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.ImportFailed" xml:space="preserve"><value>Not copied: {0} ({1})</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.MaintenanceDone" xml:space="preserve"><value>{0} stale locks, {1} temporary files, {2} packs compacted, {3} duplicate flags changed</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.Orphans" xml:space="preserve"><value>{0} unindexed photos, {1} missing photos</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.ScanDone" xml:space="preserve"><value>{0} folders, {1} indexed, {2} adopted, {3} removed</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.SequenceMissing" xml:space="preserve"><value>Sequence not found: {0}</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.VerifyDone" xml:space="preserve"><value>{0} photos checked, {1} intact, {2} hashed, {3} with problems</value><comment>long: job log line</comment></data>
|
||||
</root>
|
||||
@@ -5,4 +5,26 @@
|
||||
<resheader name="reader"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
|
||||
<resheader name="writer"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>
|
||||
<data name="Archive.Empty" xml:space="preserve"><value>Nessun archivio</value></data>
|
||||
<data name="Archive.Error.Busy" xml:space="preserve"><value>Un altro Titano ({0}) sta scrivendo qui. Riprova tra poco.</value></data>
|
||||
<data name="Archive.Error.CopyMismatch" xml:space="preserve"><value>La copia di {0} non corrisponde all'originale.</value></data>
|
||||
<data name="Archive.Error.DuplicateLost" xml:space="preserve"><value>Impossibile aggiungere {0} alla sequenza.</value></data>
|
||||
<data name="Archive.Error.NewerFormat" xml:space="preserve"><value>{0} è stato scritto da un Titano più recente (formato {1}).</value></data>
|
||||
<data name="Archive.Error.Offline" xml:space="preserve"><value>L'archivio {0} non è raggiungibile.</value></data>
|
||||
<data name="Archive.Error.ReadOnly" xml:space="preserve"><value>L'archivio {0} è di sola lettura.</value></data>
|
||||
<data name="Archive.Error.SourceChanged" xml:space="preserve"><value>{0} è cambiato dopo la creazione del piano.</value></data>
|
||||
<data name="Archive.Error.SourceGone" xml:space="preserve"><value>La scheda o cartella {0} non è più disponibile.</value></data>
|
||||
<data name="Archive.Error.StalePlan" xml:space="preserve"><value>L'archivio è cambiato dopo la creazione del piano.</value></data>
|
||||
<data name="Archive.Log.DryRun" xml:space="preserve"><value>Prova: nulla è stato modificato.</value></data>
|
||||
<data name="Archive.Log.FolderFailed" xml:space="preserve"><value>Non indicizzata: {0} ({1})</value></data>
|
||||
<data name="Archive.Log.FrameChanged" xml:space="preserve"><value>Modificato: {0}</value></data>
|
||||
<data name="Archive.Log.FrameCorrupt" xml:space="preserve"><value>Danneggiato: {0}</value></data>
|
||||
<data name="Archive.Log.FrameMissing" xml:space="preserve"><value>Mancante: {0}</value></data>
|
||||
<data name="Archive.Log.ImagesDone" xml:space="preserve"><value>{0}: {1} create, {2} riusate, {3} illeggibili</value></data>
|
||||
<data name="Archive.Log.ImportDone" xml:space="preserve"><value>{0} copiate, {1} saltate, {2} non riuscite</value></data>
|
||||
<data name="Archive.Log.ImportFailed" xml:space="preserve"><value>Non copiata: {0} ({1})</value></data>
|
||||
<data name="Archive.Log.MaintenanceDone" xml:space="preserve"><value>{0} blocchi scaduti, {1} file temporanei, {2} pacchetti compattati, {3} segni di duplicato cambiati</value></data>
|
||||
<data name="Archive.Log.Orphans" xml:space="preserve"><value>{0} foto non indicizzate, {1} foto mancanti</value></data>
|
||||
<data name="Archive.Log.ScanDone" xml:space="preserve"><value>{0} cartelle, {1} indicizzate, {2} adottate, {3} rimosse</value></data>
|
||||
<data name="Archive.Log.SequenceMissing" xml:space="preserve"><value>Sequenza non trovata: {0}</value></data>
|
||||
<data name="Archive.Log.VerifyDone" xml:space="preserve"><value>{0} foto controllate, {1} integre, {2} con impronta nuova, {3} con problemi</value></data>
|
||||
</root>
|
||||
@@ -0,0 +1,46 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Source-generated metadata for every JSON file of the archive layout.</summary>
|
||||
[JsonSourceGenerationOptions(
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNameCaseInsensitive = true,
|
||||
AllowTrailingCommas = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip)]
|
||||
[JsonSerializable(typeof(ArchiveManifest))]
|
||||
[JsonSerializable(typeof(SequenceManifest))]
|
||||
[JsonSerializable(typeof(LeaseRecord))]
|
||||
[JsonSerializable(typeof(IngestJournal))]
|
||||
[JsonSerializable(typeof(RenameJournal))]
|
||||
internal sealed partial class ArchiveJson : JsonSerializerContext;
|
||||
|
||||
/// <summary>Indented output for the small files people may open (<c>archive.json</c>, rename journals).</summary>
|
||||
[JsonSourceGenerationOptions(
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(ArchiveManifest))]
|
||||
[JsonSerializable(typeof(RenameJournal))]
|
||||
internal sealed partial class ArchiveJsonIndented : JsonSerializerContext;
|
||||
|
||||
/// <summary>
|
||||
/// Writes enum names and reads unknown names (written by a newer build) or numbers as the default value instead of
|
||||
/// failing, so a sidecar from a newer Titano is still readable.
|
||||
/// </summary>
|
||||
public sealed class TolerantEnumConverter<TEnum> : JsonConverter<TEnum>
|
||||
where TEnum : struct, Enum
|
||||
{
|
||||
public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
if (reader.TokenType == JsonTokenType.String && Enum.TryParse<TEnum>(reader.GetString(), ignoreCase: true, out var value)
|
||||
&& Enum.IsDefined(value))
|
||||
return value;
|
||||
if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) reader.Skip();
|
||||
return default;
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToString());
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Names of the fixed files and folders of the archive layout (BRIEF §7).</summary>
|
||||
public static class ArchiveLayout
|
||||
{
|
||||
/// <summary>Metadata folder at the archive root and inside every sequence folder.</summary>
|
||||
public const string MetaFolder = ".titano";
|
||||
|
||||
public const string ArchiveManifestFile = "archive.json";
|
||||
public const string IndexSnapshotFile = "index-snapshot.bin";
|
||||
public const string LocksFolder = "locks";
|
||||
|
||||
/// <summary>Verified copies waiting to be committed; on the archive volume so commits are renames.</summary>
|
||||
public const string StagingFolder = "staging";
|
||||
|
||||
/// <summary>Resumable plans of mass operations (rename).</summary>
|
||||
public const string JournalFolder = "journal";
|
||||
|
||||
public const string OriginalsFolder = "originals";
|
||||
public const string ProxiesFolder = "proxies";
|
||||
public const string ExportsFolder = "exports";
|
||||
public const string SequenceManifestFile = "sequence.json";
|
||||
public const string AnalysisFile = "analysis.json";
|
||||
public const string ProjectFile = "project.json";
|
||||
public const string ThumbsPackFile = "thumbs.pack";
|
||||
|
||||
/// <summary>Moves of an ingest batch that were started but not yet recorded in <see cref="SequenceManifestFile"/>.</summary>
|
||||
public const string IngestJournalFile = "ingest.pending.json";
|
||||
|
||||
/// <summary>Folder names inside a sequence folder that never hold a sequence of their own.</summary>
|
||||
public static bool IsReservedFolder(string name) =>
|
||||
name.Equals(MetaFolder, StringComparison.OrdinalIgnoreCase)
|
||||
|| name.Equals(OriginalsFolder, StringComparison.OrdinalIgnoreCase)
|
||||
|| name.Equals(ProxiesFolder, StringComparison.OrdinalIgnoreCase)
|
||||
|| name.Equals(ExportsFolder, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary><c>0001.jpg</c> for frame index 0: proxies are numbered from 1 like the frames of a render.</summary>
|
||||
public static string ProxyFileName(int index) => (index + 1).ToString("0000", CultureInfo.InvariantCulture) + ".jpg";
|
||||
}
|
||||
|
||||
/// <summary>Absolute paths of one archive root.</summary>
|
||||
public sealed class ArchivePaths
|
||||
{
|
||||
public ArchivePaths(string root)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(root);
|
||||
Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)));
|
||||
MetaDirectory = Path.Combine(Root, ArchiveLayout.MetaFolder);
|
||||
}
|
||||
|
||||
public string Root { get; }
|
||||
public string MetaDirectory { get; }
|
||||
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.ArchiveManifestFile);
|
||||
public string SnapshotPath => Path.Combine(MetaDirectory, ArchiveLayout.IndexSnapshotFile);
|
||||
public string LocksDirectory => Path.Combine(MetaDirectory, ArchiveLayout.LocksFolder);
|
||||
public string StagingDirectory => Path.Combine(MetaDirectory, ArchiveLayout.StagingFolder);
|
||||
public string JournalDirectory => Path.Combine(MetaDirectory, ArchiveLayout.JournalFolder);
|
||||
|
||||
/// <summary>Paths of the sequence stored in <paramref name="relativeFolder"/> (forward slashes).</summary>
|
||||
public SequencePaths Sequence(string relativeFolder) => new(this, relativeFolder);
|
||||
|
||||
/// <summary>Archive-relative path with forward slashes, as stored in the index.</summary>
|
||||
public string ToRelative(string path) => LongPath.ToPortableRelative(Root, path);
|
||||
|
||||
public string FromRelative(string relative) => LongPath.FromPortableRelative(Root, relative);
|
||||
}
|
||||
|
||||
/// <summary>Absolute paths of one sequence folder.</summary>
|
||||
public sealed class SequencePaths
|
||||
{
|
||||
internal SequencePaths(ArchivePaths archive, string relativeFolder)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(relativeFolder);
|
||||
RelativeFolder = relativeFolder.Replace('\\', '/').Trim('/');
|
||||
Folder = archive.FromRelative(RelativeFolder);
|
||||
MetaDirectory = Path.Combine(Folder, ArchiveLayout.MetaFolder);
|
||||
}
|
||||
|
||||
/// <summary>Folder relative to the archive root, forward slashes; empty when the root itself holds the photos (adopted folder).</summary>
|
||||
public string RelativeFolder { get; }
|
||||
|
||||
public string Folder { get; }
|
||||
public string MetaDirectory { get; }
|
||||
public string Originals => Path.Combine(Folder, ArchiveLayout.OriginalsFolder);
|
||||
public string Proxies => Path.Combine(Folder, ArchiveLayout.ProxiesFolder);
|
||||
public string Exports => Path.Combine(Folder, ArchiveLayout.ExportsFolder);
|
||||
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.SequenceManifestFile);
|
||||
public string AnalysisPath => Path.Combine(MetaDirectory, ArchiveLayout.AnalysisFile);
|
||||
public string ProjectPath => Path.Combine(MetaDirectory, ArchiveLayout.ProjectFile);
|
||||
public string ThumbsPackPath => Path.Combine(MetaDirectory, ArchiveLayout.ThumbsPackFile);
|
||||
public string IngestJournalPath => Path.Combine(MetaDirectory, ArchiveLayout.IngestJournalFile);
|
||||
|
||||
public string Proxy(int index) => Path.Combine(Proxies, ArchiveLayout.ProxyFileName(index));
|
||||
|
||||
/// <summary>Absolute path of a frame file given its sidecar path (relative to the sequence folder).</summary>
|
||||
public string File(string sequenceRelative) => LongPath.FromPortableRelative(Folder, sequenceRelative);
|
||||
|
||||
/// <summary>Archive-relative path of a frame file (as stored in the frames table).</summary>
|
||||
public string ArchiveRelative(string sequenceRelative) =>
|
||||
RelativeFolder.Length == 0 ? sequenceRelative : RelativeFolder + "/" + sequenceRelative;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// <c>.titano/archive.json</c> at the archive root (BRIEF §7). Fields this build does not know are kept in
|
||||
/// <see cref="Extra"/> and written back unchanged.
|
||||
/// </summary>
|
||||
public sealed record ArchiveManifest
|
||||
{
|
||||
public const int CurrentSchemaVersion = 1;
|
||||
|
||||
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
|
||||
public required Guid Id { get; init; }
|
||||
public required string Name { get; init; }
|
||||
public DateTimeOffset Created { get; init; }
|
||||
public LayoutTemplates LayoutTemplates { get; init; } = new();
|
||||
public ProxyPolicy ProxyPolicy { get; init; } = new();
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Naming templates used when frames enter the archive (BRIEF §7.1).</summary>
|
||||
public sealed record LayoutTemplates
|
||||
{
|
||||
public string Folder { get; init; } = TemplateDefaults.Folder;
|
||||
public string File { get; init; } = TemplateDefaults.File;
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Proxy generation policy (BRIEF §8.5).</summary>
|
||||
public sealed record ProxyPolicy
|
||||
{
|
||||
public const int Disabled = -1;
|
||||
public const int FullSize = 0;
|
||||
|
||||
/// <summary>Long edge in pixels: 1280, 1920 (default) or 2560; <see cref="FullSize"/> or <see cref="Disabled"/>.</summary>
|
||||
public int LongEdge { get; init; } = 1920;
|
||||
|
||||
public int Quality { get; init; } = 92;
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>A folder that holds a sequence: it has a sidecar, or photos to adopt.</summary>
|
||||
/// <param name="RelativeFolder">Archive-relative, forward slashes; empty for the root.</param>
|
||||
/// <param name="Sidecar">Stamp of <c>.titano/sequence.json</c>, null when it has none yet.</param>
|
||||
/// <param name="Photos">Stamp of the photo directory (its time changes when files are added or removed).</param>
|
||||
public sealed record SequenceFolder(string RelativeFolder, FileStamp? Sidecar, FileStamp? Photos);
|
||||
|
||||
/// <summary>
|
||||
/// Walks an archive tree (BRIEF §7). No file-system watcher is involved: callers compare the stamps with those of
|
||||
/// the previous scan. Hidden folders, symbolic links and the reserved sequence subfolders are not entered.
|
||||
/// </summary>
|
||||
public static class ArchiveScanner
|
||||
{
|
||||
private const int MaxDepth = 16;
|
||||
|
||||
private static readonly EnumerationOptions Listing = new()
|
||||
{
|
||||
IgnoreInaccessible = true,
|
||||
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System,
|
||||
RecurseSubdirectories = false,
|
||||
};
|
||||
|
||||
/// <summary>Every sequence folder under the root, ordered by relative path (ordinal) so job checkpoints are stable.</summary>
|
||||
public static IReadOnlyList<SequenceFolder> FindSequenceFolders(ArchivePaths archive, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var found = new List<SequenceFolder>();
|
||||
var pending = new Stack<(string Path, int Depth)>();
|
||||
pending.Push((archive.Root, 0));
|
||||
while (pending.Count > 0)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var (path, depth) = pending.Pop();
|
||||
var relative = archive.ToRelative(path);
|
||||
var sequence = archive.Sequence(relative == "." ? string.Empty : relative);
|
||||
var sidecar = FileStamp.Of(sequence.ManifestPath);
|
||||
var photoDirectory = PhotoDirectory(sequence);
|
||||
if (sidecar is not null || EnumeratePhotos(photoDirectory).Any())
|
||||
found.Add(new SequenceFolder(sequence.RelativeFolder, sidecar, FileStamp.Of(photoDirectory)));
|
||||
if (depth >= MaxDepth) continue;
|
||||
foreach (var child in Directory.EnumerateDirectories(path, "*", Listing))
|
||||
{
|
||||
var name = Path.GetFileName(child);
|
||||
if (!IsHidden(child, name) && !ArchiveLayout.IsReservedFolder(name)) pending.Push((child, depth + 1));
|
||||
}
|
||||
}
|
||||
found.Sort((a, b) => string.CompareOrdinal(a.RelativeFolder, b.RelativeFolder));
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary><c>originals/</c> when the folder follows the layout, otherwise the folder itself (adopted in place).</summary>
|
||||
public static string PhotoDirectory(SequencePaths sequence) => Directory.Exists(sequence.Originals) ? sequence.Originals : sequence.Folder;
|
||||
|
||||
/// <summary>Photos of a sequence folder as sidecar paths (relative to the sequence folder), ordinal order.</summary>
|
||||
public static IReadOnlyList<string> ListPhotos(SequencePaths sequence)
|
||||
{
|
||||
var directory = PhotoDirectory(sequence);
|
||||
return EnumeratePhotos(directory)
|
||||
.Select(path => Path.GetRelativePath(sequence.Folder, path).Replace('\\', '/'))
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>True for files Titano imports as photos (AppleDouble <c>._</c> companions excluded).</summary>
|
||||
public static bool IsPhoto(string fileName) =>
|
||||
!fileName.StartsWith("._", StringComparison.Ordinal) && ImageFormatInfo.IsSupportedExtension(fileName);
|
||||
|
||||
private static IEnumerable<string> EnumeratePhotos(string directory) =>
|
||||
Directory.Exists(directory)
|
||||
? Directory.EnumerateFiles(directory, "*", Listing).Where(f => IsPhoto(Path.GetFileName(f)) && !IsHidden(f, Path.GetFileName(f)))
|
||||
: [];
|
||||
|
||||
private static bool IsHidden(string path, string name) =>
|
||||
name.StartsWith('.') || (OperatingSystem.IsWindows() && File.GetAttributes(path).HasFlag(FileAttributes.Hidden));
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>The archive cannot be reached (or written) right now; nothing was changed.</summary>
|
||||
public sealed class ArchiveOfflineException(string root, ArchiveAvailability availability)
|
||||
: IOException(Loc.L(availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", root))
|
||||
{
|
||||
public string Root { get; } = root;
|
||||
public ArchiveAvailability Availability { get; } = availability;
|
||||
}
|
||||
|
||||
public enum ArchiveAvailability
|
||||
{
|
||||
Online,
|
||||
|
||||
/// <summary>Readable but not writable (read-only share, write-protected disk): browse only.</summary>
|
||||
ReadOnly,
|
||||
|
||||
/// <summary>Root unreachable (unplugged drive, disconnected share) or no longer an archive.</summary>
|
||||
Offline,
|
||||
}
|
||||
|
||||
/// <summary>Offline detection for archives on removable and network volumes (BRIEF §7).</summary>
|
||||
public static class ArchiveVolume
|
||||
{
|
||||
/// <summary>Throws <see cref="ArchiveOfflineException"/> unless the archive is online (writable).</summary>
|
||||
public static void EnsureWritable(ArchivePaths archive)
|
||||
{
|
||||
var availability = Probe(archive);
|
||||
if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(archive.Root, availability);
|
||||
}
|
||||
|
||||
/// <summary>Checks that <c>archive.json</c> is reachable and that the metadata folder accepts writes.</summary>
|
||||
public static ArchiveAvailability Probe(ArchivePaths archive)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(archive.ManifestPath)) return ArchiveAvailability.Offline;
|
||||
var probe = Path.Combine(archive.MetaDirectory, ".probe-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1, FileOptions.DeleteOnClose))
|
||||
{
|
||||
}
|
||||
return ArchiveAvailability.Online;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return ArchiveAvailability.ReadOnly;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return File.Exists(archive.ManifestPath) ? ArchiveAvailability.ReadOnly : ArchiveAvailability.Offline;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>SHA-256 of files, streamed (originals can be hundreds of megabytes).</summary>
|
||||
public static class ContentHash
|
||||
{
|
||||
internal const int BufferSize = 1 << 20;
|
||||
|
||||
/// <summary>Lower-case hex SHA-256 of the file.</summary>
|
||||
/// <param name="progress">Receives the number of bytes read since the previous call.</param>
|
||||
public static async Task<string> Sha256Async(string path, Action<long>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, BufferSize,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await Sha256Async(stream, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public static async Task<string> Sha256Async(Stream stream, Action<long>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
var buffer = new byte[BufferSize];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
progress?.Invoke(read);
|
||||
}
|
||||
return Convert.ToHexStringLower(hash.GetHashAndReset());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Size and modification time of a file, the cheap change detector of incremental scans (BRIEF §7).</summary>
|
||||
public readonly record struct FileStamp(long Size, DateTime ModifiedUtc)
|
||||
{
|
||||
/// <summary>The stamp of a file or directory, or null when it does not exist.</summary>
|
||||
public static FileStamp? Of(string path)
|
||||
{
|
||||
var file = new FileInfo(path);
|
||||
if (file.Exists) return new FileStamp(file.Length, file.LastWriteTimeUtc);
|
||||
var directory = new DirectoryInfo(path);
|
||||
return directory.Exists ? new FileStamp(0, directory.LastWriteTimeUtc) : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>A JPEG made from a frame.</summary>
|
||||
public sealed record EncodedImage(byte[] Jpeg, int Width, int Height, DecodeTier Tier);
|
||||
|
||||
/// <summary>
|
||||
/// Makes thumbnails and proxies of frames (BRIEF §8.5): the embedded preview or the managed JPEG decoder first, then
|
||||
/// the host's decode providers (WIC on Windows); orientation applied, resampled in linear light, encoded in-house.
|
||||
/// </summary>
|
||||
public sealed class FrameImageRenderer(ImageDecoder decoder)
|
||||
{
|
||||
public const int ThumbnailLongEdge = 320;
|
||||
public const int ThumbnailQuality = 80;
|
||||
|
||||
/// <summary>A 320 px q80 thumbnail, or null when no decoder can read the frame.</summary>
|
||||
public EncodedImage? Thumbnail(SequencePaths sequence, SidecarFrame frame, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var decoded = Decode(sequence.File(frame.File), frame, ThumbnailLongEdge, DecodePurpose.Thumbnail, preferPreview: true, cancellationToken);
|
||||
if (decoded is null) return null;
|
||||
using var small = Resampler.Thumbnail(decoded.Pixels, ThumbnailLongEdge);
|
||||
return Encode(small, new JpegEncodeOptions { Quality = ThumbnailQuality }, decoded.Tier);
|
||||
}
|
||||
|
||||
/// <summary>A proxy at the policy's long edge and quality with capture time and camera in its Exif; null when undecodable.</summary>
|
||||
public EncodedImage? Proxy(SequencePaths sequence, SidecarFrame frame, SidecarCamera? camera, ProxyPolicy policy, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (policy.LongEdge == ProxyPolicy.Disabled) throw new InvalidOperationException("Proxies are disabled for this archive.");
|
||||
using var decoded = Decode(sequence.File(frame.File), frame, policy.LongEdge, DecodePurpose.Proxy, preferPreview: false, cancellationToken);
|
||||
if (decoded is null) return null;
|
||||
var source = decoded.Pixels;
|
||||
var (w, h) = policy.LongEdge == ProxyPolicy.FullSize ? (source.Width, source.Height) : Resampler.FitWithin(source.Width, source.Height, policy.LongEdge);
|
||||
using var resized = Resampler.Resize(source, w, h);
|
||||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata
|
||||
{
|
||||
Make = camera?.Make,
|
||||
Model = camera?.Model,
|
||||
BodySerial = camera?.Serial,
|
||||
CapturedAt = frame.CapturedAt,
|
||||
UtcOffsetMinutes = frame.UtcOffset,
|
||||
ExposureTime = frame.Exposure,
|
||||
FNumber = frame.FNumber,
|
||||
Iso = frame.Iso,
|
||||
FocalLength = frame.Focal,
|
||||
ExposureBias = frame.Bias,
|
||||
Software = "Titano",
|
||||
});
|
||||
// The encoder takes the APP1 payload; ExifWriter returns the whole segment (marker and length first).
|
||||
return Encode(resized, new JpegEncodeOptions { Quality = policy.Quality, App1Segments = [exif[4..]] }, decoded.Tier);
|
||||
}
|
||||
|
||||
private static EncodedImage Encode(ImageBuffer pixels, JpegEncodeOptions options, DecodeTier tier) =>
|
||||
new(JpegEncoder.Encode(pixels, options), pixels.Width, pixels.Height, tier);
|
||||
|
||||
private DecodedFrame? Decode(string path, SidecarFrame frame, int maxLongEdge, DecodePurpose purpose, bool preferPreview, CancellationToken cancellationToken)
|
||||
{
|
||||
if (frame.Format == ImageFileFormat.Jpeg) return DecodeJpeg(File.ReadAllBytes(path), maxLongEdge, frame.Orientation, DecodeTier.Native);
|
||||
if (preferPreview && ReadPreview(path, frame) is { } preview) return DecodeJpeg(preview, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview);
|
||||
var request = new DecodeRequest(path, System.Math.Max(0, maxLongEdge), purpose, ApplyOrientation: true, AllowPreview: true);
|
||||
if (decoder.Decode(request, cancellationToken) is { } decoded) return new DecodedFrame(decoded.Pixels, decoded.Tier);
|
||||
return ReadPreview(path, frame) is { } fallback ? DecodeJpeg(fallback, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview) : null;
|
||||
}
|
||||
|
||||
private static DecodedFrame? DecodeJpeg(byte[] data, int maxLongEdge, int orientation, DecodeTier tier)
|
||||
{
|
||||
ImageBuffer decoded;
|
||||
try
|
||||
{
|
||||
decoded = JpegDecoder.DecodeToLinear(data, new JpegDecodeOptions { MaxLongEdge = System.Math.Max(0, maxLongEdge) }, expandGray: true);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
if (orientation is <= 1 or > 8) return new DecodedFrame(decoded, tier);
|
||||
using (decoded) return new DecodedFrame(PixelConvert.ApplyOrientation(decoded, orientation), tier);
|
||||
}
|
||||
|
||||
private static byte[]? ReadPreview(string path, SidecarFrame frame)
|
||||
{
|
||||
if (frame.PreviewOffset is not { } offset || frame.PreviewLength is not ({ } length and > 0 and < int.MaxValue)) return null;
|
||||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 4096, FileOptions.RandomAccess);
|
||||
if (offset < 0 || offset + length > stream.Length) return null;
|
||||
var data = new byte[length];
|
||||
stream.Position = offset;
|
||||
stream.ReadExactly(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private sealed record DecodedFrame(ImageBuffer Pixels, DecodeTier Tier) : IDisposable
|
||||
{
|
||||
public void Dispose() => Pixels.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
public enum FrameHealth
|
||||
{
|
||||
/// <summary>Same size and SHA-256 as recorded.</summary>
|
||||
Ok,
|
||||
|
||||
/// <summary>No hash was recorded (adopted file); it has been computed now.</summary>
|
||||
Hashed,
|
||||
|
||||
Missing,
|
||||
|
||||
/// <summary>A different size: the file was replaced or edited.</summary>
|
||||
Changed,
|
||||
|
||||
/// <summary>Same size, different content: bit rot or a damaged copy.</summary>
|
||||
Corrupt,
|
||||
}
|
||||
|
||||
/// <param name="Sha256">The hash computed now (null when the file is missing or changed size).</param>
|
||||
public sealed record FrameCheck(SidecarFrame Frame, FrameHealth Health, string? Sha256)
|
||||
{
|
||||
/// <summary>Frame flags after this check: <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> set or cleared.</summary>
|
||||
public FrameFlags Flags => Health switch
|
||||
{
|
||||
FrameHealth.Missing => (Frame.Flags & ~FrameFlags.Corrupt) | FrameFlags.Missing,
|
||||
FrameHealth.Changed or FrameHealth.Corrupt => (Frame.Flags & ~FrameFlags.Missing) | FrameFlags.Corrupt,
|
||||
_ => Frame.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt),
|
||||
};
|
||||
|
||||
/// <summary>The sidecar record after this check (flags, and the hash of a newly hashed file).</summary>
|
||||
public SidecarFrame Updated => Frame with { Flags = Flags, Sha256 = Health == FrameHealth.Hashed ? Sha256 : Frame.Sha256 };
|
||||
|
||||
public bool ChangesRecord => Updated != Frame;
|
||||
}
|
||||
|
||||
/// <summary>Re-hashes originals against their sidecar records (BRIEF §12.5 Verify).</summary>
|
||||
public static class FrameVerifier
|
||||
{
|
||||
public static async Task<FrameCheck> CheckAsync(SequencePaths sequence, SidecarFrame frame, Action<long>? bytesRead = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var info = new FileInfo(sequence.File(frame.File));
|
||||
if (!info.Exists) return new FrameCheck(frame, FrameHealth.Missing, null);
|
||||
if (info.Length != frame.Size) return new FrameCheck(frame, FrameHealth.Changed, null);
|
||||
var sha = await ContentHash.Sha256Async(info.FullName, bytesRead, cancellationToken).ConfigureAwait(false);
|
||||
var health = frame.Sha256 is null ? FrameHealth.Hashed
|
||||
: sha.Equals(frame.Sha256, StringComparison.OrdinalIgnoreCase) ? FrameHealth.Ok
|
||||
: FrameHealth.Corrupt;
|
||||
return new FrameCheck(frame, health, sha);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// <c>.titano/index-snapshot.bin</c> (BRIEF §7): every sidecar of the archive in one file, so a rebuild reads one file
|
||||
/// sequentially instead of thousands of small ones over a share. Optional and never authoritative: an entry is used
|
||||
/// only when its sidecar still has the recorded size and time, and a file whose trailing SHA-256 does not match is
|
||||
/// ignored entirely.
|
||||
/// </summary>
|
||||
/// <remarks>Layout: "TISN", version (u16), reserved (u16), then per sidecar: folder length (u16), UTF-8 folder,
|
||||
/// size (i64), modified UTC ticks (i64), length (i32), sidecar bytes; finally the SHA-256 of everything before it.</remarks>
|
||||
public sealed class IndexSnapshot
|
||||
{
|
||||
private const ushort Version = 1;
|
||||
private static ReadOnlySpan<byte> Magic => "TISN"u8;
|
||||
|
||||
private readonly Dictionary<string, (FileStamp Stamp, byte[] Bytes)> _entries;
|
||||
|
||||
private IndexSnapshot(Dictionary<string, (FileStamp, byte[])> entries) => _entries = entries;
|
||||
|
||||
public int Count => _entries.Count;
|
||||
|
||||
/// <summary>The snapshot at <paramref name="path"/>, or null when it is absent, damaged or of another version.</summary>
|
||||
public static IndexSnapshot? TryLoad(string path)
|
||||
{
|
||||
byte[] data;
|
||||
try
|
||||
{
|
||||
if (!File.Exists(path)) return null;
|
||||
data = File.ReadAllBytes(path);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
if (data.Length < 8 + 32 || !data.AsSpan(0, 4).SequenceEqual(Magic) || BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan(4)) != Version)
|
||||
return null;
|
||||
var body = data.AsSpan(0, data.Length - 32);
|
||||
if (!SHA256.HashData(body).AsSpan().SequenceEqual(data.AsSpan(data.Length - 32))) return null;
|
||||
|
||||
var entries = new Dictionary<string, (FileStamp, byte[])>(StringComparer.Ordinal);
|
||||
var position = 8;
|
||||
while (position < body.Length)
|
||||
{
|
||||
if (position + 2 > body.Length) return null;
|
||||
var folderLength = BinaryPrimitives.ReadUInt16LittleEndian(body[position..]);
|
||||
position += 2;
|
||||
if (position + folderLength + 20 > body.Length) return null;
|
||||
var folder = Encoding.UTF8.GetString(body.Slice(position, folderLength));
|
||||
position += folderLength;
|
||||
var size = BinaryPrimitives.ReadInt64LittleEndian(body[position..]);
|
||||
var ticks = BinaryPrimitives.ReadInt64LittleEndian(body[(position + 8)..]);
|
||||
var length = BinaryPrimitives.ReadInt32LittleEndian(body[(position + 16)..]);
|
||||
position += 20;
|
||||
if (length < 0 || ticks < 0 || ticks > DateTime.MaxValue.Ticks || position + length > body.Length) return null;
|
||||
entries[folder] = (new FileStamp(size, new DateTime(ticks, DateTimeKind.Utc)), body.Slice(position, length).ToArray());
|
||||
position += length;
|
||||
}
|
||||
return new IndexSnapshot(entries);
|
||||
}
|
||||
|
||||
/// <summary>The recorded sidecar bytes of <paramref name="folder"/> when its current stamp still matches.</summary>
|
||||
public byte[]? TryGet(string folder, FileStamp current) =>
|
||||
_entries.TryGetValue(folder, out var entry) && entry.Stamp == current ? entry.Bytes : null;
|
||||
|
||||
/// <summary>Streams a new snapshot to a temporary file; <see cref="Commit"/> replaces the old one atomically.</summary>
|
||||
public sealed class Writer : IDisposable
|
||||
{
|
||||
private readonly string _path;
|
||||
private readonly string _temp;
|
||||
private readonly FileStream _stream;
|
||||
private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
private bool _committed;
|
||||
|
||||
public Writer(string path)
|
||||
{
|
||||
_path = path;
|
||||
_temp = AtomicFile.TempPathFor(path);
|
||||
_stream = new FileStream(_temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16);
|
||||
Span<byte> header = stackalloc byte[8];
|
||||
header.Clear();
|
||||
Magic.CopyTo(header);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
|
||||
Write(header);
|
||||
}
|
||||
|
||||
public void Add(string folder, FileStamp stamp, ReadOnlySpan<byte> sidecar)
|
||||
{
|
||||
var name = Encoding.UTF8.GetBytes(folder);
|
||||
Span<byte> head = stackalloc byte[2];
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(head, checked((ushort)name.Length));
|
||||
Write(head);
|
||||
Write(name);
|
||||
Span<byte> fields = stackalloc byte[20];
|
||||
BinaryPrimitives.WriteInt64LittleEndian(fields, stamp.Size);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(fields[8..], stamp.ModifiedUtc.Ticks);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(fields[16..], sidecar.Length);
|
||||
Write(fields);
|
||||
Write(sidecar);
|
||||
}
|
||||
|
||||
public void Commit()
|
||||
{
|
||||
_stream.Write(_hash.GetHashAndReset());
|
||||
_stream.Dispose();
|
||||
AtomicFile.Commit(_temp, _path);
|
||||
_committed = true;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_stream.Dispose();
|
||||
_hash.Dispose();
|
||||
if (!_committed && File.Exists(_temp)) File.Delete(_temp);
|
||||
}
|
||||
|
||||
private void Write(ReadOnlySpan<byte> data)
|
||||
{
|
||||
_hash.AppendData(data);
|
||||
_stream.Write(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Write-ahead record of an ingest batch (<c>.titano/ingest.pending.json</c>): the frames whose files are about to be
|
||||
/// moved into the sequence folder. Written before the moves and removed once <c>sequence.json</c> lists them, so a
|
||||
/// crash in between is repaired by <see cref="Recover"/> instead of leaving unindexed originals behind.
|
||||
/// </summary>
|
||||
public sealed record IngestJournal(IReadOnlyList<SidecarFrame> Frames)
|
||||
{
|
||||
public static void Write(SequencePaths sequence, IngestJournal journal)
|
||||
{
|
||||
Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
AtomicFile.WriteAllBytes(sequence.IngestJournalPath, JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJson.Default.IngestJournal));
|
||||
}
|
||||
|
||||
public static void Delete(SequencePaths sequence)
|
||||
{
|
||||
if (File.Exists(sequence.IngestJournalPath)) File.Delete(sequence.IngestJournalPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes an interrupted batch: frames whose file reached the sequence folder (same size) but are missing from
|
||||
/// <paramref name="manifest"/> are added in capture order. Returns the manifest to write, or null when nothing has
|
||||
/// to change. The journal is left in place; <see cref="Delete"/> it once the result is written.
|
||||
/// </summary>
|
||||
public static SequenceManifest? Recover(SequencePaths sequence, SequenceManifest manifest)
|
||||
{
|
||||
if (!File.Exists(sequence.IngestJournalPath)) return null;
|
||||
IngestJournal? journal;
|
||||
try
|
||||
{
|
||||
journal = JsonSerializer.Deserialize(File.ReadAllBytes(sequence.IngestJournalPath), ArchiveJson.Default.IngestJournal);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
journal = null; // torn while being written: nothing was moved yet
|
||||
}
|
||||
if (journal is null) return null;
|
||||
var known = manifest.Frames.Select(f => f.Id).ToHashSet();
|
||||
var moved = journal.Frames.Where(f => !known.Contains(f.Id) && FileHasSize(sequence.File(f.File), f.Size)).ToList();
|
||||
return moved.Count == 0 ? null : SequenceFrames.Insert(manifest, moved);
|
||||
}
|
||||
|
||||
private static bool FileHasSize(string path, long size)
|
||||
{
|
||||
var info = new FileInfo(path);
|
||||
return info.Exists && info.Length == size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Content of a lease file: who holds it and until when.</summary>
|
||||
public sealed record LeaseRecord(string Token, string Owner, string Host, int ProcessId, DateTimeOffset Acquired, DateTimeOffset Expires);
|
||||
|
||||
/// <summary>The lease is held by someone else and did not free up in time.</summary>
|
||||
public sealed class LeaseUnavailableException(string name, LeaseRecord? holder)
|
||||
: IOException(Loc.L("Archive.Error.Busy", holder?.Owner ?? "?"))
|
||||
{
|
||||
public string Name { get; } = name;
|
||||
public LeaseRecord? Holder { get; } = holder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cross-process exclusive lease stored as <c>.titano/locks/<name>.lease</c> (BRIEF §7), for writers that share
|
||||
/// an archive over a network (server and client). Creation uses create-new semantics, which SMB and local file
|
||||
/// systems both make atomic. A lease past its expiry is stale (its holder crashed or lost the share) and is taken
|
||||
/// over by renaming it aside, which only one contender can do. The holder renews it in the background while held.
|
||||
/// </summary>
|
||||
public sealed class LeaseFile : IDisposable
|
||||
{
|
||||
public static readonly TimeSpan DefaultDuration = TimeSpan.FromMinutes(2);
|
||||
|
||||
private const string Extension = ".lease";
|
||||
private const string StaleExtension = ".stale";
|
||||
private readonly Lock _gate = new();
|
||||
private readonly Timer _renewal;
|
||||
private LeaseRecord _record;
|
||||
private bool _released;
|
||||
|
||||
private LeaseFile(string path, string name, LeaseRecord record, TimeSpan duration)
|
||||
{
|
||||
Path = path;
|
||||
Name = name;
|
||||
_record = record;
|
||||
Duration = duration;
|
||||
var period = duration / 3;
|
||||
_renewal = new Timer(_ => Renew(), null, period, period);
|
||||
}
|
||||
|
||||
public string Path { get; }
|
||||
public string Name { get; }
|
||||
public TimeSpan Duration { get; }
|
||||
public LeaseRecord Record => _record;
|
||||
|
||||
/// <summary>False once a renewal found the lease taken over (it expired while this process was stalled).</summary>
|
||||
public bool IsHeld { get; private set; } = true;
|
||||
|
||||
/// <summary>Takes the lease when it is free or stale; null when someone else holds it.</summary>
|
||||
public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null)
|
||||
{
|
||||
var length = duration ?? DefaultDuration;
|
||||
var path = PathFor(locksDirectory, name);
|
||||
Directory.CreateDirectory(locksDirectory);
|
||||
for (var attempt = 0; attempt < 3; attempt++)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var record = new LeaseRecord(Guid.NewGuid().ToString("N"), owner, Environment.MachineName, Environment.ProcessId, now, now + length);
|
||||
if (TryCreate(path, record)) return new LeaseFile(path, name, record, length);
|
||||
if (!IsStale(path, now)) return null;
|
||||
BreakStale(path, now);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Waits up to <paramref name="wait"/> for the lease, polling with back-off.</summary>
|
||||
/// <exception cref="LeaseUnavailableException">Still held by someone else after <paramref name="wait"/>.</exception>
|
||||
public static async Task<LeaseFile> AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait,
|
||||
TimeSpan? duration = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + wait;
|
||||
var delay = TimeSpan.FromMilliseconds(20);
|
||||
while (true)
|
||||
{
|
||||
if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease;
|
||||
if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name)));
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, 500));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The current holder of a lease, or null when free or unreadable.</summary>
|
||||
public static LeaseRecord? Read(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.LeaseRecord);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Removes every stale lease in <paramref name="locksDirectory"/> (maintenance); returns their names.</summary>
|
||||
public static IReadOnlyList<string> CleanupStale(string locksDirectory, bool dryRun)
|
||||
{
|
||||
if (!Directory.Exists(locksDirectory)) return [];
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var removed = new List<string>();
|
||||
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + Extension))
|
||||
{
|
||||
if (!IsStale(path, now)) continue;
|
||||
removed.Add(System.IO.Path.GetFileNameWithoutExtension(path));
|
||||
if (!dryRun) BreakStale(path, now);
|
||||
}
|
||||
if (!dryRun)
|
||||
{
|
||||
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + StaleExtension))
|
||||
{
|
||||
if (File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime) TryDelete(path);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>Extends the expiry; false when the lease is no longer ours.</summary>
|
||||
public bool Renew()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_released || !IsHeld) return false;
|
||||
if (Read(Path)?.Token != _record.Token)
|
||||
{
|
||||
IsHeld = false;
|
||||
Log.Warning("archive", $"Lease {Name} was taken over by another writer");
|
||||
return false;
|
||||
}
|
||||
var renewed = _record with { Expires = DateTimeOffset.UtcNow + Duration };
|
||||
try
|
||||
{
|
||||
// In-place rewrite: a rename would open a window in which the lease looks free.
|
||||
using var fs = new FileStream(Path, FileMode.Truncate, FileAccess.Write, FileShare.Read);
|
||||
fs.Write(JsonSerializer.SerializeToUtf8Bytes(renewed, ArchiveJson.Default.LeaseRecord));
|
||||
fs.Flush(flushToDisk: true);
|
||||
_record = renewed;
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Runs on a timer thread: an escaping exception would end the process.
|
||||
Log.Warning("archive", $"Lease {Name} could not be renewed", ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_renewal.Dispose();
|
||||
lock (_gate)
|
||||
{
|
||||
if (_released) return;
|
||||
_released = true;
|
||||
if (!IsHeld || Read(Path)?.Token != _record.Token) return;
|
||||
try
|
||||
{
|
||||
File.Delete(Path);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// It expires on its own.
|
||||
Log.Warning("archive", $"Lease {Name} could not be released", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal static string PathFor(string locksDirectory, string name) => System.IO.Path.Combine(locksDirectory, name + Extension);
|
||||
|
||||
private static bool TryCreate(string path, LeaseRecord record)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
|
||||
fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
|
||||
fs.Flush(flushToDisk: true);
|
||||
return true;
|
||||
}
|
||||
catch (IOException) when (File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Expired, or unreadable for longer than a lease lasts (a holder that crashed while creating it).</summary>
|
||||
private static bool IsStale(string path, DateTimeOffset now)
|
||||
{
|
||||
if (Read(path) is { } record) return record.Expires < now;
|
||||
try
|
||||
{
|
||||
return File.Exists(path) && File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static void BreakStale(string path, DateTimeOffset now)
|
||||
{
|
||||
var aside = path + "." + Guid.NewGuid().ToString("N")[..8] + StaleExtension;
|
||||
try
|
||||
{
|
||||
File.Move(path, aside);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return; // another contender broke it first
|
||||
}
|
||||
if (IsStale(aside, now))
|
||||
{
|
||||
TryDelete(aside);
|
||||
return;
|
||||
}
|
||||
// A contender renewed or re-created the lease between our check and the rename: give it back.
|
||||
try
|
||||
{
|
||||
File.Move(aside, path, overwrite: false);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
TryDelete(aside);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// A leftover file is harmless; maintenance removes it.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>A sidecar written by a newer Titano: it can be read, but this build refuses to rewrite it.</summary>
|
||||
public sealed class NewerFormatException(string path, int version)
|
||||
: InvalidOperationException(Loc.L("Archive.Error.NewerFormat", System.IO.Path.GetFileName(path), version))
|
||||
{
|
||||
public string Path { get; } = path;
|
||||
public int Version { get; } = version;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads and writes the JSON files of the layout. Reading tolerates unknown fields and enum values; every write
|
||||
/// goes through <see cref="AtomicFile"/> (temp → fsync → rename), so a crash leaves the previous file intact.
|
||||
/// </summary>
|
||||
public static class ManifestStore
|
||||
{
|
||||
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
|
||||
public static ArchiveManifest ReadArchive(string path) => Parse(File.ReadAllBytes(path), path, ArchiveJson.Default.ArchiveManifest);
|
||||
|
||||
public static void WriteArchive(string path, ArchiveManifest manifest)
|
||||
{
|
||||
RefuseNewer(path, manifest.SchemaVersion, ArchiveManifest.CurrentSchemaVersion);
|
||||
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJsonIndented.Default.ArchiveManifest));
|
||||
}
|
||||
|
||||
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
|
||||
public static SequenceManifest ReadSequence(string path) => ParseSequence(File.ReadAllBytes(path), path);
|
||||
|
||||
/// <summary>Parses sidecar bytes (from disk or from the index snapshot).</summary>
|
||||
public static SequenceManifest ParseSequence(ReadOnlySpan<byte> utf8, string source) => Parse(utf8, source, ArchiveJson.Default.SequenceManifest);
|
||||
|
||||
/// <summary>Replaces the sidecar atomically. Compact JSON: a 100k-frame sidecar stays around 35 MB.</summary>
|
||||
public static void WriteSequence(string path, SequenceManifest manifest)
|
||||
{
|
||||
RefuseNewer(path, manifest.SchemaVersion, SequenceManifest.CurrentSchemaVersion);
|
||||
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the sidecar only if none exists (rename without replace), which makes it the claim on a new
|
||||
/// sequence folder between processes sharing the archive. False when another sidecar is already there.
|
||||
/// </summary>
|
||||
public static bool TryCreateSequence(string path, SequenceManifest manifest)
|
||||
{
|
||||
if (File.Exists(path)) return false;
|
||||
var temp = AtomicFile.TempPathFor(path);
|
||||
File.WriteAllBytes(temp, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) fs.Flush(flushToDisk: true);
|
||||
File.Move(temp, path, overwrite: false);
|
||||
return true;
|
||||
}
|
||||
catch (IOException) when (File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temp)) File.Delete(temp);
|
||||
}
|
||||
}
|
||||
|
||||
private static T Parse<T>(ReadOnlySpan<byte> utf8, string source, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(utf8, typeInfo) ?? throw new InvalidDataException($"{source} is empty.");
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{source} is not a valid Titano file: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static void RefuseNewer(string path, int version, int supported)
|
||||
{
|
||||
if (version > supported) throw new NewerFormatException(path, version);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>What the planner needs to know about a sequence to rename it.</summary>
|
||||
public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens);
|
||||
|
||||
/// <summary>One sequence's change: its folder, its title, or both.</summary>
|
||||
public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle)
|
||||
{
|
||||
public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>The exact changes a mass rename will make (its dry run); applying it makes exactly these.</summary>
|
||||
public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList<RenameStep> Steps);
|
||||
|
||||
/// <summary>A plan being applied, with the steps already completed; lets an interrupted rename resume (BRIEF §7).</summary>
|
||||
public sealed record RenameJournal(RenamePlan Plan, IReadOnlyList<Guid> Completed)
|
||||
{
|
||||
public static string PathFor(ArchivePaths archive, Guid planId) => Path.Combine(archive.JournalDirectory, $"rename-{planId:N}.json");
|
||||
|
||||
public static void Write(ArchivePaths archive, RenameJournal journal)
|
||||
{
|
||||
Directory.CreateDirectory(archive.JournalDirectory);
|
||||
AtomicFile.WriteAllBytes(PathFor(archive, journal.Plan.Id), JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJsonIndented.Default.RenameJournal));
|
||||
}
|
||||
|
||||
/// <summary>Journals left by interrupted renames.</summary>
|
||||
public static IReadOnlyList<RenameJournal> ListPending(ArchivePaths archive)
|
||||
{
|
||||
if (!Directory.Exists(archive.JournalDirectory)) return [];
|
||||
var list = new List<RenameJournal>();
|
||||
foreach (var path in Directory.EnumerateFiles(archive.JournalDirectory, "rename-*.json").Order(StringComparer.Ordinal))
|
||||
{
|
||||
if (JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.RenameJournal) is { } journal) list.Add(journal);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static void Delete(ArchivePaths archive, Guid planId) => File.Delete(PathFor(archive, planId));
|
||||
}
|
||||
|
||||
/// <summary>Builds rename plans from folder/title templates (BRIEF §7.1, §12.5).</summary>
|
||||
public static class RenamePlanner
|
||||
{
|
||||
/// <summary>
|
||||
/// Expands <paramref name="titleTemplate"/> (where <c>{title}</c> is the current title) and then
|
||||
/// <paramref name="folderTemplate"/> (where <c>{title}</c> is the new title) for every subject. A target folder
|
||||
/// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"….
|
||||
/// Null templates leave that part unchanged; subjects without a change produce no step.
|
||||
/// </summary>
|
||||
/// <param name="occupied">Every sequence folder of the archive (renamed or not).</param>
|
||||
public static RenamePlan Plan(Guid archiveId, IReadOnlyList<RenameSubject> subjects, string? folderTemplate, string? titleTemplate,
|
||||
IEnumerable<string> occupied, Func<string, bool> existsOnDisk)
|
||||
{
|
||||
var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate);
|
||||
var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate);
|
||||
var claimed = new HashSet<string>(occupied, StringComparer.OrdinalIgnoreCase);
|
||||
var steps = new List<RenameStep>();
|
||||
foreach (var s in subjects)
|
||||
{
|
||||
var context = new NamingContext
|
||||
{
|
||||
CapturedAt = s.FirstCapture ?? DateTime.Today,
|
||||
SessionStart = s.FirstCapture ?? DateTime.Today,
|
||||
CameraMake = s.CameraMake,
|
||||
CameraModel = s.CameraModel,
|
||||
Lens = s.Lens,
|
||||
Title = s.Title,
|
||||
};
|
||||
var newTitle = title is null ? s.Title : Blank(title.ExpandFile(context)) ?? s.Title;
|
||||
var newFolder = s.Folder;
|
||||
// The archive root itself (a folder adopted in place) cannot move; only its title changes.
|
||||
if (folder is not null && s.Folder.Length > 0 && Blank(folder.ExpandFolder(context with { Title = newTitle })) is { } expanded
|
||||
&& !expanded.Equals(s.Folder, StringComparison.Ordinal))
|
||||
{
|
||||
newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk);
|
||||
claimed.Add(newFolder);
|
||||
}
|
||||
if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle));
|
||||
}
|
||||
return new RenamePlan(Guid.CreateVersion7(), archiveId, steps);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted,
|
||||
/// a case-only change goes through a temporary name (case-insensitive file systems).
|
||||
/// </summary>
|
||||
/// <exception cref="IOException">The target is occupied by something else, or neither folder exists.</exception>
|
||||
public static void MoveFolder(ArchivePaths archive, RenameStep step)
|
||||
{
|
||||
if (!step.MovesFolder) return;
|
||||
var from = archive.FromRelative(step.FromFolder);
|
||||
var to = archive.FromRelative(step.ToFolder);
|
||||
var caseOnly = step.FromFolder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase);
|
||||
if (Directory.Exists(to) && (!caseOnly || HasExactName(to)))
|
||||
{
|
||||
if (Directory.Exists(from) && !caseOnly) throw new IOException($"Rename target {step.ToFolder} already exists.");
|
||||
return;
|
||||
}
|
||||
if (!Directory.Exists(from)) throw new DirectoryNotFoundException($"Sequence folder {step.FromFolder} is missing.");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
|
||||
if (caseOnly)
|
||||
{
|
||||
var temp = from + ".rename-" + Guid.NewGuid().ToString("N")[..8];
|
||||
Directory.Move(from, temp);
|
||||
Directory.Move(temp, to);
|
||||
}
|
||||
else
|
||||
{
|
||||
Directory.Move(from, to);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool HasExactName(string path) =>
|
||||
Directory.EnumerateDirectories(Path.GetDirectoryName(path)!, Path.GetFileName(path))
|
||||
.Any(d => Path.GetFileName(d).Equals(Path.GetFileName(path), StringComparison.Ordinal));
|
||||
|
||||
private static string? Blank(string value) => string.IsNullOrWhiteSpace(value) ? null : value;
|
||||
|
||||
private static string Unique(string folder, HashSet<string> claimed, Func<string, bool> existsOnDisk)
|
||||
{
|
||||
var candidate = folder;
|
||||
for (var n = 2; claimed.Contains(candidate) || existsOnDisk(candidate); n++)
|
||||
candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})");
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Files of a sequence folder that the sidecar does not list, and sidecar frames whose file is gone.</summary>
|
||||
public sealed record SequenceOrphans(string Folder, IReadOnlyList<string> UnindexedFiles, IReadOnlyList<SidecarFrame> MissingFiles)
|
||||
{
|
||||
public bool IsEmpty => UnindexedFiles.Count == 0 && MissingFiles.Count == 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds and reconciles sidecars from the photos found in a folder (BRIEF §7, §12.5 "Adopt folder"). Only light
|
||||
/// metadata is read; originals are never moved, renamed or rewritten. Hashes are left empty for a verify to fill.
|
||||
/// </summary>
|
||||
public static class SequenceAdopter
|
||||
{
|
||||
/// <summary>A new sidecar for a folder without one. A folder outside the layout (no <c>originals/</c>) is titled by its name.</summary>
|
||||
public static SequenceManifest Build(Guid id, SequencePaths sequence, string? createdBy, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (frames, first) = ReadFrames(sequence, ArchiveScanner.ListPhotos(sequence), cancellationToken);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var inLayout = Directory.Exists(sequence.Originals);
|
||||
var manifest = new SequenceManifest
|
||||
{
|
||||
Id = id,
|
||||
Title = inLayout ? null : Path.GetFileName(Path.TrimEndingDirectorySeparator(sequence.Folder)),
|
||||
Camera = first is null ? null : new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial },
|
||||
Lens = first?.LensModel,
|
||||
CreatedBy = createdBy,
|
||||
Created = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
return SequenceFrames.Insert(manifest, frames);
|
||||
}
|
||||
|
||||
/// <summary>Unindexed photos and missing frame files of one sequence.</summary>
|
||||
public static SequenceOrphans FindOrphans(SequencePaths sequence, SequenceManifest manifest)
|
||||
{
|
||||
var listed = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal);
|
||||
var unindexed = ArchiveScanner.ListPhotos(sequence).Where(f => !listed.Contains(f)).ToList();
|
||||
var missing = manifest.Frames.Where(f => !File.Exists(sequence.File(f.File))).ToList();
|
||||
return new SequenceOrphans(sequence.RelativeFolder, unindexed, missing);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Brings a sidecar in line with its folder: unindexed photos are added (in capture order), frames whose file is
|
||||
/// gone get <see cref="FrameFlags.Missing"/> and frames whose file came back lose it. Null when nothing changed.
|
||||
/// </summary>
|
||||
public static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var orphans = FindOrphans(sequence, manifest);
|
||||
var missing = orphans.MissingFiles.Select(f => f.Id).ToHashSet();
|
||||
var flagChanges = new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>>();
|
||||
foreach (var frame in manifest.Frames)
|
||||
{
|
||||
var isMissing = missing.Contains(frame.Id);
|
||||
if (isMissing == frame.Flags.HasFlag(FrameFlags.Missing)) continue;
|
||||
flagChanges[frame.Id] = f => f with { Flags = isMissing ? f.Flags | FrameFlags.Missing : f.Flags & ~FrameFlags.Missing };
|
||||
}
|
||||
if (flagChanges.Count == 0 && orphans.UnindexedFiles.Count == 0) return null;
|
||||
var updated = flagChanges.Count == 0 ? manifest : SequenceFrames.Update(manifest, flagChanges);
|
||||
var (added, first) = ReadFrames(sequence, orphans.UnindexedFiles, cancellationToken);
|
||||
if (updated.Camera is null && first is not null)
|
||||
updated = updated with { Camera = new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, Lens = updated.Lens ?? first.LensModel };
|
||||
return SequenceFrames.Insert(updated, added);
|
||||
}
|
||||
|
||||
private static (List<SidecarFrame> Frames, FrameMetadata? First) ReadFrames(SequencePaths sequence, IReadOnlyList<string> files, CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = new List<SidecarFrame>(files.Count);
|
||||
FrameMetadata? first = null;
|
||||
foreach (var file in files)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var path = sequence.File(file);
|
||||
try
|
||||
{
|
||||
var metadata = MetadataReader.Read(path, cancellationToken);
|
||||
if (first is null || (metadata.CapturedAt < first.CapturedAt && metadata.Make is not null)) first = metadata;
|
||||
frames.Add(SequenceFrames.FromMetadata(Guid.CreateVersion7(), file, Path.GetFileName(file), new FileInfo(path).Length, null, metadata));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"Skipped unreadable photo {sequence.ArchiveRelative(file)}", ex);
|
||||
}
|
||||
}
|
||||
return (frames, first);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Edits of the frame list of a <see cref="SequenceManifest"/>; frames stay in capture order with gap-free indexes.</summary>
|
||||
public static class SequenceFrames
|
||||
{
|
||||
/// <summary>Capture time (unknown last), then file name.</summary>
|
||||
public static readonly IComparer<SidecarFrame> CaptureOrder = Comparer<SidecarFrame>.Create((a, b) =>
|
||||
{
|
||||
var byTime = (a.CapturedAt ?? DateTime.MaxValue).CompareTo(b.CapturedAt ?? DateTime.MaxValue);
|
||||
return byTime != 0 ? byTime : string.CompareOrdinal(a.File, b.File);
|
||||
});
|
||||
|
||||
/// <summary>A frame record from light metadata.</summary>
|
||||
/// <param name="file">Path relative to the sequence folder.</param>
|
||||
/// <param name="fallbackCapture">Used when the file records no capture time of its own (only a file-system date).</param>
|
||||
public static SidecarFrame FromMetadata(Guid id, string file, string? original, long size, string? sha256, FrameMetadata metadata,
|
||||
DateTime? fallbackCapture = null)
|
||||
{
|
||||
var useFallback = fallbackCapture is not null && metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown;
|
||||
return new SidecarFrame
|
||||
{
|
||||
Id = id,
|
||||
File = file,
|
||||
Original = original,
|
||||
Size = size,
|
||||
Sha256 = sha256?.ToLowerInvariant(),
|
||||
CapturedAt = useFallback ? DateTime.SpecifyKind(fallbackCapture!.Value, DateTimeKind.Unspecified) : metadata.CapturedAt,
|
||||
UtcOffset = useFallback ? null : metadata.UtcOffsetMinutes,
|
||||
TimeSource = useFallback ? TimestampSource.Unknown : metadata.TimestampSource,
|
||||
Exposure = metadata.ExposureTime,
|
||||
FNumber = metadata.FNumber,
|
||||
Iso = metadata.Iso,
|
||||
Focal = metadata.FocalLength,
|
||||
Bias = metadata.ExposureBias,
|
||||
Wb = metadata.WhiteBalanceMode,
|
||||
Orientation = metadata.Orientation,
|
||||
W = metadata.Width,
|
||||
H = metadata.Height,
|
||||
Format = metadata.Format,
|
||||
PreviewOffset = metadata.LargestPreview?.Offset,
|
||||
PreviewLength = metadata.LargestPreview?.Length,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges <paramref name="added"/> into the frame list by capture time (existing frames keep their relative order),
|
||||
/// renumbers indexes and bumps the frames version.
|
||||
/// </summary>
|
||||
public static SequenceManifest Insert(SequenceManifest manifest, IReadOnlyList<SidecarFrame> added)
|
||||
{
|
||||
if (added.Count == 0) return manifest;
|
||||
var incoming = added.Order(CaptureOrder).ToList();
|
||||
var merged = new List<SidecarFrame>(manifest.Frames.Count + incoming.Count);
|
||||
var next = 0;
|
||||
foreach (var frame in manifest.Frames)
|
||||
{
|
||||
while (next < incoming.Count && CaptureOrder.Compare(incoming[next], frame) < 0) merged.Add(incoming[next++]);
|
||||
merged.Add(frame);
|
||||
}
|
||||
while (next < incoming.Count) merged.Add(incoming[next++]);
|
||||
return WithFrames(manifest, merged);
|
||||
}
|
||||
|
||||
/// <summary>Replaces the frame list (indexes renumbered) and bumps the frames version.</summary>
|
||||
public static SequenceManifest WithFrames(SequenceManifest manifest, IReadOnlyList<SidecarFrame> frames)
|
||||
{
|
||||
var renumbered = new SidecarFrame[frames.Count];
|
||||
for (var i = 0; i < frames.Count; i++) renumbered[i] = frames[i].I == i ? frames[i] : frames[i] with { I = i };
|
||||
return manifest with
|
||||
{
|
||||
Frames = renumbered,
|
||||
Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Applies per-frame changes by id (flags, hashes); frames not in <paramref name="changes"/> are kept.</summary>
|
||||
public static SequenceManifest Update(SequenceManifest manifest, IReadOnlyDictionary<Guid, Func<SidecarFrame, SidecarFrame>> changes)
|
||||
{
|
||||
var frames = manifest.Frames.Select(f => changes.TryGetValue(f.Id, out var change) ? change(f) : f).ToList();
|
||||
return manifest with { Frames = frames, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// <c>.titano/sequence.json</c>, the canonical record of one sequence (BRIEF §7). The database is rebuilt from it,
|
||||
/// so it carries the ids of the sequence and of every frame. Fields this build does not know are kept in
|
||||
/// <see cref="Extra"/> and written back unchanged.
|
||||
/// </summary>
|
||||
public sealed record SequenceManifest
|
||||
{
|
||||
public const int CurrentSchemaVersion = 1;
|
||||
|
||||
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
|
||||
public required Guid Id { get; init; }
|
||||
public string? Title { get; init; }
|
||||
public SidecarCamera? Camera { get; init; }
|
||||
public string? Lens { get; init; }
|
||||
|
||||
/// <summary>Frames in capture order; <see cref="SidecarFrame.I"/> equals the position.</summary>
|
||||
public IReadOnlyList<SidecarFrame> Frames { get; init; } = [];
|
||||
|
||||
public IReadOnlyList<string> Tags { get; init; } = [];
|
||||
public int Rating { get; init; }
|
||||
public string? Notes { get; init; }
|
||||
|
||||
/// <summary><see cref="SequenceStatus"/> bits, stored as a number so bits of newer builds survive.</summary>
|
||||
public SequenceStatus Status { get; init; }
|
||||
|
||||
public ManifestVersions Versions { get; init; } = new();
|
||||
|
||||
/// <summary>User id or client name that created the sequence.</summary>
|
||||
public string? CreatedBy { get; init; }
|
||||
|
||||
public DateTimeOffset Created { get; init; }
|
||||
public DateTimeOffset UpdatedAt { get; init; }
|
||||
public DateTimeOffset? VerifiedAt { get; init; }
|
||||
|
||||
/// <summary>Session keys of automatic grouping (uploads, card imports) that resolve to this sequence.</summary>
|
||||
public IReadOnlyList<string> SessionKeys { get; init; } = [];
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
public sealed record SidecarCamera
|
||||
{
|
||||
public string? Make { get; init; }
|
||||
public string? Model { get; init; }
|
||||
public string? Serial { get; init; }
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Counters the sync protocol compares (BRIEF §14.5).</summary>
|
||||
public sealed record ManifestVersions
|
||||
{
|
||||
public int Settings { get; init; }
|
||||
public int Frames { get; init; }
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>One frame of <see cref="SequenceManifest"/>; property names follow BRIEF §7.</summary>
|
||||
public sealed record SidecarFrame
|
||||
{
|
||||
public required Guid Id { get; init; }
|
||||
|
||||
/// <summary>Index in capture order, from 0.</summary>
|
||||
public int I { get; init; }
|
||||
|
||||
/// <summary>Path relative to the sequence folder, forward slashes (<c>originals/…</c> for imported frames).</summary>
|
||||
public required string File { get; init; }
|
||||
|
||||
/// <summary>Name the file had on the card or upload source.</summary>
|
||||
public string? Original { get; init; }
|
||||
|
||||
public long Size { get; init; }
|
||||
|
||||
/// <summary>Lower-case hex SHA-256; null until a verify hashes an adopted file.</summary>
|
||||
public string? Sha256 { get; init; }
|
||||
|
||||
/// <summary>Camera wall-clock time with sub-seconds.</summary>
|
||||
public DateTime? CapturedAt { get; init; }
|
||||
|
||||
public int? UtcOffset { get; init; }
|
||||
|
||||
[JsonConverter(typeof(TolerantEnumConverter<TimestampSource>))]
|
||||
public TimestampSource TimeSource { get; init; }
|
||||
|
||||
public double? Exposure { get; init; }
|
||||
public double? FNumber { get; init; }
|
||||
public int? Iso { get; init; }
|
||||
public double? Focal { get; init; }
|
||||
public double? Bias { get; init; }
|
||||
public int? Wb { get; init; }
|
||||
public int Orientation { get; init; } = 1;
|
||||
public int W { get; init; }
|
||||
public int H { get; init; }
|
||||
|
||||
[JsonConverter(typeof(TolerantEnumConverter<ImageFileFormat>))]
|
||||
public ImageFileFormat Format { get; init; }
|
||||
|
||||
public long? PreviewOffset { get; init; }
|
||||
public long? PreviewLength { get; init; }
|
||||
|
||||
/// <summary><see cref="FrameFlags"/> bits, stored as a number so bits of newer builds survive.</summary>
|
||||
public FrameFlags Flags { get; init; }
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
using System.Buffers.Binary;
|
||||
using Titano.Core.Hashing;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>One thumbnail of a pack: where its JPEG lies and which source it was made from.</summary>
|
||||
/// <param name="SourceHash">32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale.</param>
|
||||
public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc);
|
||||
|
||||
/// <summary>
|
||||
/// <c>thumbs.pack</c>: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence
|
||||
/// instead of thousands. Little-endian blocks after a 16-byte header:
|
||||
/// <list type="bullet">
|
||||
/// <item><c>TENT</c> entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes.</item>
|
||||
/// <item><c>TIDX</c> index: count, one row per frame (latest entry wins), CRC-32 of the rows.</item>
|
||||
/// <item><c>TFTR</c> footer (last 16 bytes): offset of the index block and its CRC.</item>
|
||||
/// </list>
|
||||
/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index
|
||||
/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall
|
||||
/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it).
|
||||
/// </summary>
|
||||
public static class ThumbPack
|
||||
{
|
||||
public const int HashLength = 32;
|
||||
internal const int HeaderLength = 16;
|
||||
internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4;
|
||||
internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4;
|
||||
internal const int FooterLength = 16;
|
||||
internal const ushort Version = 1;
|
||||
|
||||
/// <summary>Largest thumbnail accepted; anything bigger in a header is corruption.</summary>
|
||||
internal const int MaxEntryLength = 16 << 20;
|
||||
|
||||
internal static ReadOnlySpan<byte> Magic => "TTPK"u8;
|
||||
internal static ReadOnlySpan<byte> EntryTag => "TENT"u8;
|
||||
internal static ReadOnlySpan<byte> IndexTag => "TIDX"u8;
|
||||
internal static ReadOnlySpan<byte> FooterTag => "TFTR"u8;
|
||||
|
||||
internal static void WriteHeader(Stream stream)
|
||||
{
|
||||
Span<byte> header = stackalloc byte[HeaderLength];
|
||||
header.Clear();
|
||||
Magic.CopyTo(header);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
|
||||
stream.Write(header);
|
||||
}
|
||||
|
||||
internal static bool HasValidHeader(Stream stream)
|
||||
{
|
||||
if (stream.Length < HeaderLength) return false;
|
||||
Span<byte> header = stackalloc byte[HeaderLength];
|
||||
stream.Position = 0;
|
||||
stream.ReadExactly(header);
|
||||
return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The latest entry per frame and where the valid content ends: after the footer when the index is intact
|
||||
/// (<paramref name="indexOffset"/> = start of that index), otherwise after the last complete block.
|
||||
/// </summary>
|
||||
internal static Dictionary<int, ThumbEntry> Load(Stream stream, out long validEnd, out long? indexOffset)
|
||||
{
|
||||
indexOffset = null;
|
||||
if (TryLoadIndex(stream, out var entries, out var offset))
|
||||
{
|
||||
validEnd = stream.Length;
|
||||
indexOffset = offset;
|
||||
return entries;
|
||||
}
|
||||
return Scan(stream, out validEnd);
|
||||
}
|
||||
|
||||
private static bool TryLoadIndex(Stream stream, out Dictionary<int, ThumbEntry> entries, out long indexOffset)
|
||||
{
|
||||
entries = [];
|
||||
indexOffset = 0;
|
||||
var length = stream.Length;
|
||||
if (length < HeaderLength + 12 + FooterLength) return false;
|
||||
Span<byte> footer = stackalloc byte[FooterLength];
|
||||
stream.Position = length - FooterLength;
|
||||
stream.ReadExactly(footer);
|
||||
if (!footer[..4].SequenceEqual(FooterTag)) return false;
|
||||
indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]);
|
||||
if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false;
|
||||
if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false;
|
||||
|
||||
Span<byte> head = stackalloc byte[8];
|
||||
stream.Position = indexOffset;
|
||||
stream.ReadExactly(head);
|
||||
var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]);
|
||||
if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false;
|
||||
var rows = new byte[count * IndexRowLength + 4];
|
||||
stream.ReadExactly(rows);
|
||||
if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var row = rows.AsSpan(i * IndexRowLength, IndexRowLength);
|
||||
var entry = new ThumbEntry(
|
||||
BinaryPrimitives.ReadInt32LittleEndian(row),
|
||||
BinaryPrimitives.ReadInt64LittleEndian(row[4..]),
|
||||
BinaryPrimitives.ReadInt32LittleEndian(row[12..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(row[16..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(row[18..]),
|
||||
row.Slice(20, HashLength).ToArray(),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..]));
|
||||
if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false;
|
||||
entries[entry.FrameIndex] = entry;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Walks block headers from the start; stops at the first incomplete or unknown block.</summary>
|
||||
private static Dictionary<int, ThumbEntry> Scan(Stream stream, out long validEnd)
|
||||
{
|
||||
var entries = new Dictionary<int, ThumbEntry>();
|
||||
var length = stream.Length;
|
||||
var position = (long)HeaderLength;
|
||||
var header = new byte[EntryHeaderLength];
|
||||
while (position + 4 <= length)
|
||||
{
|
||||
stream.Position = position;
|
||||
stream.ReadExactly(header.AsSpan(0, 4));
|
||||
var tag = header.AsSpan(0, 4);
|
||||
long next;
|
||||
if (tag.SequenceEqual(EntryTag))
|
||||
{
|
||||
if (position + EntryHeaderLength > length) break;
|
||||
stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4));
|
||||
var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength));
|
||||
if (size is < 0 or > MaxEntryLength) break;
|
||||
next = position + EntryHeaderLength + size;
|
||||
if (next > length) break;
|
||||
var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
|
||||
entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size,
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)),
|
||||
header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength)));
|
||||
}
|
||||
else if (tag.SequenceEqual(IndexTag))
|
||||
{
|
||||
if (position + 8 > length) break;
|
||||
stream.ReadExactly(header.AsSpan(4, 4));
|
||||
var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
|
||||
if (count < 0) break;
|
||||
next = position + 8 + (long)count * IndexRowLength + 4;
|
||||
if (next > length) break;
|
||||
}
|
||||
else if (tag.SequenceEqual(FooterTag))
|
||||
{
|
||||
next = position + FooterLength;
|
||||
if (next > length) break;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
position = next;
|
||||
}
|
||||
validEnd = position;
|
||||
return entries;
|
||||
}
|
||||
|
||||
/// <summary>Writes one entry block at the stream position and returns its entry.</summary>
|
||||
internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
|
||||
{
|
||||
var crc = Crc32.Compute(jpeg);
|
||||
Span<byte> header = stackalloc byte[EntryHeaderLength];
|
||||
EntryTag.CopyTo(header);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height);
|
||||
sourceHash.CopyTo(header[12..]);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc);
|
||||
var payload = stream.Position + EntryHeaderLength;
|
||||
stream.Write(header);
|
||||
stream.Write(jpeg);
|
||||
return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc);
|
||||
}
|
||||
|
||||
internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection<ThumbEntry> entries)
|
||||
{
|
||||
var indexOffset = stream.Position;
|
||||
var block = new byte[8 + entries.Count * IndexRowLength + 4];
|
||||
IndexTag.CopyTo(block);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count);
|
||||
var i = 0;
|
||||
foreach (var e in entries.OrderBy(e => e.FrameIndex))
|
||||
{
|
||||
var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height);
|
||||
e.SourceHash.CopyTo(row[20..]);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc);
|
||||
}
|
||||
var rowsLength = entries.Count * IndexRowLength;
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength)));
|
||||
stream.Write(block);
|
||||
|
||||
Span<byte> footer = stackalloc byte[FooterLength];
|
||||
FooterTag.CopyTo(footer);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12]));
|
||||
stream.Write(footer);
|
||||
}
|
||||
|
||||
/// <summary>Size of a pack holding only <paramref name="entries"/>; the rest of a file is reclaimable garbage.</summary>
|
||||
public static long LiveBytes(IEnumerable<ThumbEntry> entries)
|
||||
{
|
||||
long count = 0, payload = 0;
|
||||
foreach (var e in entries)
|
||||
{
|
||||
count++;
|
||||
payload += EntryHeaderLength + e.Length;
|
||||
}
|
||||
return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength;
|
||||
}
|
||||
|
||||
/// <summary>A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known.</summary>
|
||||
public static byte[] Fingerprint(string? sha256, string file, long size) =>
|
||||
sha256 is { Length: HashLength * 2 }
|
||||
? Convert.FromHexString(sha256)
|
||||
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
|
||||
}
|
||||
|
||||
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
|
||||
public sealed class ThumbPackReader : IDisposable
|
||||
{
|
||||
private readonly FileStream? _stream;
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
private ThumbPackReader(FileStream? stream, Dictionary<int, ThumbEntry> entries)
|
||||
{
|
||||
_stream = stream;
|
||||
Entries = entries;
|
||||
}
|
||||
|
||||
/// <summary>The latest entry of every frame that has one.</summary>
|
||||
public IReadOnlyDictionary<int, ThumbEntry> Entries { get; }
|
||||
|
||||
/// <summary>Bytes held by replaced entries (reclaimed by <see cref="ThumbPackWriter.Compact"/>).</summary>
|
||||
public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values));
|
||||
|
||||
public long Length => _stream?.Length ?? 0;
|
||||
|
||||
/// <summary>Opens a pack; a missing or damaged pack reads as empty.</summary>
|
||||
public static ThumbPackReader Open(string path)
|
||||
{
|
||||
if (!File.Exists(path)) return new ThumbPackReader(null, []);
|
||||
var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess);
|
||||
try
|
||||
{
|
||||
var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : [];
|
||||
return new ThumbPackReader(stream, entries);
|
||||
}
|
||||
catch
|
||||
{
|
||||
stream.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The JPEG of a frame, or null when absent or damaged (CRC mismatch).</summary>
|
||||
public byte[]? Read(int frameIndex)
|
||||
{
|
||||
if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null;
|
||||
var data = new byte[entry.Length];
|
||||
lock (_gate)
|
||||
{
|
||||
if (entry.Offset + entry.Length > _stream.Length) return null;
|
||||
_stream.Position = entry.Offset;
|
||||
_stream.ReadExactly(data);
|
||||
}
|
||||
return Crc32.Compute(data) == entry.Crc ? data : null;
|
||||
}
|
||||
|
||||
public void Dispose() => _stream?.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it).</summary>
|
||||
public sealed class ThumbPackWriter : IDisposable
|
||||
{
|
||||
private readonly FileStream _stream;
|
||||
private readonly Dictionary<int, ThumbEntry> _entries;
|
||||
private long? _indexOffset;
|
||||
private bool _dirty;
|
||||
|
||||
private ThumbPackWriter(FileStream stream, Dictionary<int, ThumbEntry> entries, long? indexOffset)
|
||||
{
|
||||
_stream = stream;
|
||||
_entries = entries;
|
||||
_indexOffset = indexOffset;
|
||||
}
|
||||
|
||||
public IReadOnlyDictionary<int, ThumbEntry> Entries => _entries;
|
||||
|
||||
/// <summary>Opens or creates a pack, dropping an incomplete tail left by a crash.</summary>
|
||||
public static ThumbPackWriter Open(string path)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
|
||||
var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16);
|
||||
try
|
||||
{
|
||||
if (!ThumbPack.HasValidHeader(stream))
|
||||
{
|
||||
// Not a pack (or a header torn by a crash): thumbnails are regenerable, start over.
|
||||
stream.SetLength(0);
|
||||
ThumbPack.WriteHeader(stream);
|
||||
return new ThumbPackWriter(stream, [], null);
|
||||
}
|
||||
var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset);
|
||||
if (validEnd < stream.Length) stream.SetLength(validEnd);
|
||||
return new ThumbPackWriter(stream, entries, indexOffset);
|
||||
}
|
||||
catch
|
||||
{
|
||||
stream.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Appends a thumbnail; it replaces any earlier entry of the frame once committed.</summary>
|
||||
public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(frameIndex);
|
||||
if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash));
|
||||
if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg));
|
||||
if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width));
|
||||
// The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk).
|
||||
_stream.Position = _indexOffset ?? _stream.Length;
|
||||
if (_indexOffset is { } offset) _stream.SetLength(offset);
|
||||
_indexOffset = null;
|
||||
|
||||
_entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg);
|
||||
_dirty = true;
|
||||
}
|
||||
|
||||
/// <summary>Writes the index and footer and flushes to disk.</summary>
|
||||
public void Commit()
|
||||
{
|
||||
if (!_dirty) return;
|
||||
_stream.Position = _stream.Length;
|
||||
var indexOffset = _stream.Position;
|
||||
ThumbPack.WriteIndexAndFooter(_stream, _entries.Values);
|
||||
_stream.Flush(flushToDisk: true);
|
||||
_indexOffset = indexOffset;
|
||||
_dirty = false;
|
||||
}
|
||||
|
||||
/// <summary>Rewrites the pack with only its live entries (atomically, so readers keep a consistent file).</summary>
|
||||
public static void Compact(string path)
|
||||
{
|
||||
var temp = IO.AtomicFile.TempPathFor(path);
|
||||
try
|
||||
{
|
||||
using (var source = ThumbPackReader.Open(path))
|
||||
using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
|
||||
{
|
||||
ThumbPack.WriteHeader(target);
|
||||
var entries = new List<ThumbEntry>(source.Entries.Count);
|
||||
foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex))
|
||||
{
|
||||
if (source.Read(entry.FrameIndex) is { } jpeg)
|
||||
entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg));
|
||||
}
|
||||
ThumbPack.WriteIndexAndFooter(target, entries);
|
||||
}
|
||||
IO.AtomicFile.Commit(temp, path);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temp)) File.Delete(temp);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
Commit();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_stream.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class ArchiveFolderTests(TestContext context)
|
||||
{
|
||||
private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
|
||||
|
||||
private ArchivePaths NewArchive() => new(context.CreateTempDirectory("archive"));
|
||||
|
||||
[Test]
|
||||
public void SequenceFoldersAreFoundInLayoutAndAdoptedForms()
|
||||
{
|
||||
var archive = NewArchive();
|
||||
TestPhotos.WriteSeries(Path.Combine(archive.Root, "2026", "2026-09-28 Canon EOS R5", "originals"), Start, 3, TimeSpan.FromSeconds(10));
|
||||
TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour"), Start, 2, TimeSpan.FromSeconds(10));
|
||||
TestPhotos.WriteSeries(Path.Combine(archive.Root, "Trips", "Harbour", "selects"), Start, 1, TimeSpan.FromSeconds(10));
|
||||
TestPhotos.WriteSeries(Path.Combine(archive.Root, ".hidden"), Start, 1, TimeSpan.FromSeconds(10));
|
||||
File.WriteAllBytes(Path.Combine(archive.Root, "Trips", "._IMG_0001.JPG"), [1]);
|
||||
Directory.CreateDirectory(Path.Combine(archive.Root, "empty"));
|
||||
|
||||
var folders = ArchiveScanner.FindSequenceFolders(archive);
|
||||
Assert.SequenceEqual(["2026/2026-09-28 Canon EOS R5", "Trips/Harbour", "Trips/Harbour/selects"], folders.Select(f => f.RelativeFolder));
|
||||
Assert.True(folders.All(f => f.Sidecar is null && f.Photos is not null));
|
||||
Assert.SequenceEqual(["originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG"],
|
||||
ArchiveScanner.ListPhotos(archive.Sequence("2026/2026-09-28 Canon EOS R5")));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AdoptionBuildsASidecarInCaptureOrderWithoutTouchingPhotos()
|
||||
{
|
||||
var archive = NewArchive();
|
||||
var folder = Path.Combine(archive.Root, "Harbour sunset");
|
||||
// File names run against capture order: the sidecar follows capture time.
|
||||
TestPhotos.Write(Path.Combine(folder, "a.jpg"), Start.AddSeconds(20), 1);
|
||||
TestPhotos.Write(Path.Combine(folder, "b.jpg"), Start, 2);
|
||||
TestPhotos.Write(Path.Combine(folder, "c.jpg"), Start.AddSeconds(10), 3);
|
||||
var before = Directory.GetFiles(folder).ToDictionary(f => f, File.ReadAllBytes);
|
||||
|
||||
var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), archive.Sequence("Harbour sunset"), "test");
|
||||
Assert.SequenceEqual(["b.jpg", "c.jpg", "a.jpg"], manifest.Frames.Select(f => f.File));
|
||||
Assert.SequenceEqual([0, 1, 2], manifest.Frames.Select(f => f.I));
|
||||
Assert.Equal("Harbour sunset", manifest.Title);
|
||||
Assert.Equal("Canon EOS R5", manifest.Camera!.Model);
|
||||
Assert.True(manifest.Frames.All(f => f.Sha256 is null && f.Size > 0 && f.Format == ImageFileFormat.Jpeg && f.W == 48));
|
||||
foreach (var (path, bytes) in before) Assert.BytesEqual(bytes, File.ReadAllBytes(path));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReconcileIndexesNewPhotosAndFlagsMissingOnes()
|
||||
{
|
||||
var archive = NewArchive();
|
||||
var sequence = archive.Sequence("s");
|
||||
var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10));
|
||||
var manifest = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null);
|
||||
Assert.Null(SequenceAdopter.Reconcile(sequence, manifest));
|
||||
|
||||
File.Delete(files[1]);
|
||||
TestPhotos.Write(Path.Combine(sequence.Originals, "IMG_0000.JPG"), Start.AddSeconds(-10), 9);
|
||||
var orphans = SequenceAdopter.FindOrphans(sequence, manifest);
|
||||
Assert.SequenceEqual(["originals/IMG_0000.JPG"], orphans.UnindexedFiles);
|
||||
Assert.SequenceEqual(["originals/IMG_0002.JPG"], orphans.MissingFiles.Select(f => f.File));
|
||||
|
||||
var reconciled = SequenceAdopter.Reconcile(sequence, manifest)!;
|
||||
Assert.SequenceEqual(["originals/IMG_0000.JPG", "originals/IMG_0001.JPG", "originals/IMG_0002.JPG", "originals/IMG_0003.JPG", "originals/IMG_0004.JPG"],
|
||||
reconciled.Frames.Select(f => f.File));
|
||||
Assert.Equal(FrameFlags.Missing, reconciled.Frames[2].Flags);
|
||||
Assert.Equal(manifest.Frames[0].Id, reconciled.Frames[1].Id);
|
||||
|
||||
TestPhotos.Write(files[1], Start.AddSeconds(10), 2);
|
||||
var restored = SequenceAdopter.Reconcile(sequence, reconciled)!;
|
||||
Assert.Equal(FrameFlags.None, restored.Frames[2].Flags);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task VerifyDetectsAFlippedByteAMissingFileAndAChangedFile()
|
||||
{
|
||||
var archive = NewArchive();
|
||||
var sequence = archive.Sequence("s");
|
||||
var files = TestPhotos.WriteSeries(sequence.Originals, Start, 4, TimeSpan.FromSeconds(10));
|
||||
var adopted = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null);
|
||||
var hashed = new List<SidecarFrame>();
|
||||
foreach (var frame in adopted.Frames)
|
||||
{
|
||||
var check = await FrameVerifier.CheckAsync(sequence, frame);
|
||||
Assert.Equal(FrameHealth.Hashed, check.Health);
|
||||
hashed.Add(check.Updated);
|
||||
}
|
||||
|
||||
var bytes = File.ReadAllBytes(files[0]);
|
||||
bytes[bytes.Length / 2] ^= 0x01;
|
||||
File.WriteAllBytes(files[0], bytes);
|
||||
File.Delete(files[1]);
|
||||
File.AppendAllText(files[2], "edit");
|
||||
|
||||
var health = new List<FrameHealth>();
|
||||
foreach (var frame in hashed) health.Add((await FrameVerifier.CheckAsync(sequence, frame)).Health);
|
||||
Assert.SequenceEqual([FrameHealth.Corrupt, FrameHealth.Missing, FrameHealth.Changed, FrameHealth.Ok], health);
|
||||
var corrupt = await FrameVerifier.CheckAsync(sequence, hashed[0]);
|
||||
Assert.Equal(FrameFlags.Corrupt, corrupt.Flags);
|
||||
Assert.Equal(hashed[0].Sha256, corrupt.Updated.Sha256);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheIndexSnapshotIsUsedOnlyForUnchangedSidecars()
|
||||
{
|
||||
var archive = NewArchive();
|
||||
Directory.CreateDirectory(archive.MetaDirectory);
|
||||
var stampA = new FileStamp(10, new DateTime(2026, 9, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||
using (var writer = new IndexSnapshot.Writer(archive.SnapshotPath))
|
||||
{
|
||||
writer.Add("a", stampA, "{\"a\":1}"u8);
|
||||
writer.Add("b/ç", new FileStamp(20, stampA.ModifiedUtc), "{\"b\":2}"u8);
|
||||
writer.Commit();
|
||||
}
|
||||
var snapshot = IndexSnapshot.TryLoad(archive.SnapshotPath)!;
|
||||
Assert.Equal(2, snapshot.Count);
|
||||
Assert.BytesEqual("{\"a\":1}"u8, snapshot.TryGet("a", stampA));
|
||||
Assert.Null(snapshot.TryGet("a", stampA with { Size = 11 }));
|
||||
Assert.NotNull(snapshot.TryGet("b/ç", new FileStamp(20, stampA.ModifiedUtc)));
|
||||
|
||||
var bytes = File.ReadAllBytes(archive.SnapshotPath);
|
||||
bytes[12] ^= 0x40;
|
||||
File.WriteAllBytes(archive.SnapshotPath, bytes);
|
||||
Assert.Null(IndexSnapshot.TryLoad(archive.SnapshotPath));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RenamePlansAvoidTakenFoldersAndMoveIdempotently()
|
||||
{
|
||||
var archive = NewArchive();
|
||||
Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old a"));
|
||||
Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "old b"));
|
||||
Directory.CreateDirectory(Path.Combine(archive.Root, "2026", "2026-09-28 Sunset"));
|
||||
var a = new RenameSubject(Guid.NewGuid(), "2026/old a", null, Start, "Canon", "Canon EOS R5", null);
|
||||
var b = new RenameSubject(Guid.NewGuid(), "2026/old b", "Dawn", Start, "Canon", "Canon EOS R5", null);
|
||||
var plan = RenamePlanner.Plan(Guid.NewGuid(), [a, b], "{year}/{date} {title}", "Sunset",
|
||||
["2026/old a", "2026/old b"], f => Directory.Exists(archive.FromRelative(f)));
|
||||
Assert.SequenceEqual(["2026/2026-09-28 Sunset (2)", "2026/2026-09-28 Sunset (3)"], plan.Steps.Select(s => s.ToFolder));
|
||||
Assert.True(plan.Steps.All(s => s.ToTitle == "Sunset"));
|
||||
Assert.Equal("Dawn", plan.Steps[1].FromTitle);
|
||||
|
||||
RenamePlanner.MoveFolder(archive, plan.Steps[0]);
|
||||
RenamePlanner.MoveFolder(archive, plan.Steps[0]);
|
||||
Assert.False(Directory.Exists(archive.FromRelative("2026/old a")));
|
||||
Assert.True(Directory.Exists(archive.FromRelative("2026/2026-09-28 Sunset (2)")));
|
||||
|
||||
var titleOnly = RenamePlanner.Plan(Guid.NewGuid(), [b], null, "{title} morning", ["2026/old b"], _ => false);
|
||||
Assert.Equal("2026/old b", titleOnly.Steps.Single().ToFolder);
|
||||
Assert.Equal("Dawn morning", titleOnly.Steps.Single().ToTitle);
|
||||
Assert.Empty(RenamePlanner.Plan(Guid.NewGuid(), [b], "2026/old b", null, ["2026/old b"], _ => true).Steps);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ThumbnailsAndProxiesAreRenderedFromTheFrame()
|
||||
{
|
||||
var archive = NewArchive();
|
||||
var sequence = archive.Sequence("s");
|
||||
Directory.CreateDirectory(sequence.Originals);
|
||||
File.WriteAllBytes(Path.Combine(sequence.Originals, "big.jpg"), TestPhotos.Jpeg(Start, 7, width: 640, height: 480, orientation: 6));
|
||||
var frame = SequenceAdopter.Build(Guid.CreateVersion7(), sequence, null).Frames.Single();
|
||||
var renderer = new FrameImageRenderer(new ImageDecoder([]));
|
||||
|
||||
var thumb = renderer.Thumbnail(sequence, frame)!;
|
||||
Assert.Equal((240, 320), (thumb.Width, thumb.Height));
|
||||
Assert.Equal(DecodeTier.Native, thumb.Tier);
|
||||
Assert.Equal(240, JpegDecoder.ReadInfo(thumb.Jpeg).Width);
|
||||
|
||||
var proxy = renderer.Proxy(sequence, frame, new SidecarCamera { Make = "Canon", Model = "Canon EOS R5" }, new ProxyPolicy { LongEdge = 400 })!;
|
||||
Assert.Equal((300, 400), (proxy.Width, proxy.Height));
|
||||
var exif = MetadataReader.Read(new MemoryStream(proxy.Jpeg), "p.jpg");
|
||||
Assert.Equal(Start, exif.CapturedAt);
|
||||
Assert.Equal("Canon EOS R5", exif.Model);
|
||||
Assert.Equal(1, exif.Orientation);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class LeaseFileTests(TestContext context)
|
||||
{
|
||||
private static void WriteRecord(string path, LeaseRecord record) =>
|
||||
File.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
|
||||
|
||||
[Test]
|
||||
public void ALeaseIsExclusiveUntilReleased()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
using (var a = LeaseFile.TryAcquire(locks, "seq-1", "server"))
|
||||
{
|
||||
Assert.NotNull(a);
|
||||
Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client"));
|
||||
using var other = LeaseFile.TryAcquire(locks, "seq-2", "client");
|
||||
Assert.NotNull(other);
|
||||
}
|
||||
using var b = LeaseFile.TryAcquire(locks, "seq-1", "client");
|
||||
Assert.NotNull(b);
|
||||
Assert.Equal("client", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExpiredLeaseOfACrashedHolderIsTakenOver()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
var past = DateTimeOffset.UtcNow.AddMinutes(-10);
|
||||
WriteRecord(LeaseFile.PathFor(locks, "seq-1"), new LeaseRecord("dead", "crashed", "pc", 1, past, past.AddMinutes(2)));
|
||||
using var lease = LeaseFile.TryAcquire(locks, "seq-1", "survivor");
|
||||
Assert.NotNull(lease);
|
||||
Assert.Equal("survivor", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Owner);
|
||||
Assert.Count(1, Directory.GetFiles(locks));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AHeldLeaseIsRenewedPastItsDuration()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
using var lease = LeaseFile.TryAcquire(locks, "seq-1", "server", TimeSpan.FromMilliseconds(300));
|
||||
Assert.NotNull(lease);
|
||||
await Task.Delay(900);
|
||||
Assert.True(lease.IsHeld);
|
||||
Assert.Greater((LeaseFile.Read(lease.Path)!.Expires - DateTimeOffset.UtcNow).TotalMilliseconds, 0);
|
||||
Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client", TimeSpan.FromMilliseconds(300)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AFreshEmptyLeaseFileIsNotStaleButAnOldOneIs()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
var path = LeaseFile.PathFor(locks, "seq-1");
|
||||
File.WriteAllBytes(path, []);
|
||||
Assert.Null(LeaseFile.TryAcquire(locks, "seq-1", "client"));
|
||||
File.SetLastWriteTimeUtc(path, DateTime.UtcNow.AddHours(-1));
|
||||
using var lease = LeaseFile.TryAcquire(locks, "seq-1", "client");
|
||||
Assert.NotNull(lease);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ALostLeaseIsDetectedAndNotDeletedByItsFormerHolder()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
var lease = LeaseFile.TryAcquire(locks, "seq-1", "slow");
|
||||
Assert.NotNull(lease);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
WriteRecord(lease.Path, new LeaseRecord("thief", "other", "pc", 2, now, now.AddMinutes(2)));
|
||||
Assert.False(lease.Renew());
|
||||
Assert.False(lease.IsHeld);
|
||||
lease.Dispose();
|
||||
Assert.Equal("thief", LeaseFile.Read(LeaseFile.PathFor(locks, "seq-1"))!.Token);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task WaitingForABusyLeaseTimesOutWithItsHolder()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
using var held = LeaseFile.TryAcquire(locks, "seq-1", "server");
|
||||
var error = await Assert.ThrowsAsync<LeaseUnavailableException>(() =>
|
||||
LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromMilliseconds(200)));
|
||||
Assert.Equal("server", error.Holder!.Owner);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task WaitingForALeaseSucceedsOnceItIsReleased()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
var held = LeaseFile.TryAcquire(locks, "seq-1", "server")!;
|
||||
var waiting = LeaseFile.AcquireAsync(locks, "seq-1", "client", TimeSpan.FromSeconds(10));
|
||||
await Task.Delay(100);
|
||||
held.Dispose();
|
||||
using var lease = await waiting;
|
||||
Assert.Equal("client", lease.Record.Owner);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CleanupRemovesOnlyStaleLeasesAndItsDryRunNothing()
|
||||
{
|
||||
var locks = context.CreateTempDirectory();
|
||||
var past = DateTimeOffset.UtcNow.AddMinutes(-10);
|
||||
WriteRecord(LeaseFile.PathFor(locks, "dead"), new LeaseRecord("t", "crashed", "pc", 1, past, past.AddMinutes(1)));
|
||||
using var live = LeaseFile.TryAcquire(locks, "live", "server");
|
||||
Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: true));
|
||||
Assert.Count(2, Directory.GetFiles(locks));
|
||||
Assert.SequenceEqual(["dead"], LeaseFile.CleanupStale(locks, dryRun: false));
|
||||
Assert.SequenceEqual([LeaseFile.PathFor(locks, "live")], Directory.GetFiles(locks));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class ManifestStoreTests(TestContext context)
|
||||
{
|
||||
private static SequenceManifest Sample() => new()
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Title = "Harbour sunset",
|
||||
Camera = new SidecarCamera { Make = "Canon", Model = "Canon EOS R5", Serial = "0431" },
|
||||
Lens = "RF24-70mm F2.8 L IS USM",
|
||||
Frames =
|
||||
[
|
||||
new SidecarFrame
|
||||
{
|
||||
Id = Guid.CreateVersion7(), I = 0, File = "originals/2026-09-28_18-02-11_0001.CR2", Original = "IMG_0001.CR2", Size = 25_000_000,
|
||||
Sha256 = new string('a', 64), CapturedAt = new DateTime(2026, 9, 28, 18, 2, 11, 250), UtcOffset = 120,
|
||||
TimeSource = TimestampSource.ExifOriginal, Exposure = 1.0 / 250, FNumber = 8, Iso = 100, Focal = 24, Bias = -0.3, Wb = 0,
|
||||
Orientation = 6, W = 8192, H = 5464, Format = ImageFileFormat.Cr2, PreviewOffset = 1234, PreviewLength = 567_890,
|
||||
Flags = FrameFlags.Excluded,
|
||||
},
|
||||
],
|
||||
Tags = ["sunset", "harbour"],
|
||||
Rating = 4,
|
||||
Notes = "Windy",
|
||||
Status = SequenceStatus.ThumbsBuilt | SequenceStatus.Verified,
|
||||
Versions = new ManifestVersions { Settings = 3, Frames = 7 },
|
||||
CreatedBy = "alberto",
|
||||
Created = new DateTimeOffset(2026, 9, 28, 20, 0, 0, TimeSpan.Zero),
|
||||
UpdatedAt = new DateTimeOffset(2026, 9, 29, 8, 0, 0, TimeSpan.Zero),
|
||||
VerifiedAt = new DateTimeOffset(2026, 9, 30, 8, 0, 0, TimeSpan.Zero),
|
||||
SessionKeys = ["card-1234abcd-20260928-180211"],
|
||||
};
|
||||
|
||||
private string SidecarPath() => Path.Combine(context.CreateTempDirectory(), ".titano", "sequence.json");
|
||||
|
||||
private static string Canonical(SequenceManifest m) => JsonSerializer.Serialize(m, ArchiveJson.Default.SequenceManifest);
|
||||
|
||||
[Test]
|
||||
public void SequenceSidecarRoundTripsEveryField()
|
||||
{
|
||||
var path = SidecarPath();
|
||||
var original = Sample();
|
||||
ManifestStore.WriteSequence(path, original);
|
||||
var read = ManifestStore.ReadSequence(path);
|
||||
Assert.Equal(Canonical(original), Canonical(read));
|
||||
Assert.Equal(original.Frames[0], read.Frames[0] with { Extra = original.Frames[0].Extra });
|
||||
var json = File.ReadAllText(path);
|
||||
foreach (var field in new[] { "\"i\":", "\"file\":", "\"sha256\":", "\"capturedAt\":", "\"fNumber\":", "\"w\":", "\"previewOffset\":", "\"flags\":" })
|
||||
Assert.Contains(field, json);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UnknownFieldsAndEnumValuesFromANewerBuildAreReadAndKept()
|
||||
{
|
||||
var path = SidecarPath();
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
var id = Guid.CreateVersion7();
|
||||
var frameId = Guid.CreateVersion7();
|
||||
File.WriteAllText(path, $$"""
|
||||
{
|
||||
"schemaVersion": 1, "id": "{{id}}", "title": "Comet", "futureBlock": { "a": [1, 2] },
|
||||
"camera": { "make": "Nikon", "sensorTemp": 31 },
|
||||
"frames": [ { "id": "{{frameId}}", "i": 0, "file": "x.nef", "size": 10, "format": "Nef2030", "timeSource": "Atomic",
|
||||
"flags": 4096, "aiScore": 0.9 } ],
|
||||
"status": 1024, "versions": { "settings": 1, "frames": 2, "renders": 5 },
|
||||
}
|
||||
""");
|
||||
var read = ManifestStore.ReadSequence(path);
|
||||
Assert.Equal(id, read.Id);
|
||||
Assert.Equal(ImageFileFormat.Unknown, read.Frames[0].Format);
|
||||
Assert.Equal(TimestampSource.Unknown, read.Frames[0].TimeSource);
|
||||
Assert.Equal((FrameFlags)4096, read.Frames[0].Flags);
|
||||
|
||||
ManifestStore.WriteSequence(path, read with { Title = "Comet 2026" });
|
||||
var json = File.ReadAllText(path);
|
||||
foreach (var kept in new[] { "futureBlock", "sensorTemp", "aiScore", "renders", "4096", "1024" }) Assert.Contains(kept, json);
|
||||
Assert.Equal("Comet 2026", ManifestStore.ReadSequence(path).Title);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANewerSchemaIsReadableButNeverRewritten()
|
||||
{
|
||||
var path = SidecarPath();
|
||||
ManifestStore.WriteSequence(path, Sample());
|
||||
var newer = ManifestStore.ReadSequence(path) with { SchemaVersion = SequenceManifest.CurrentSchemaVersion + 1 };
|
||||
var before = File.ReadAllBytes(path);
|
||||
Assert.Throws<NewerFormatException>(() => ManifestStore.WriteSequence(path, newer));
|
||||
Assert.BytesEqual(before, File.ReadAllBytes(path));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void DamagedJsonIsReportedAsInvalidData()
|
||||
{
|
||||
var path = SidecarPath();
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
File.WriteAllText(path, "{ \"id\": ");
|
||||
Assert.Throws<InvalidDataException>(() => ManifestStore.ReadSequence(path));
|
||||
File.WriteAllText(path, "{ \"title\": \"no id\" }");
|
||||
Assert.Throws<InvalidDataException>(() => ManifestStore.ReadSequence(path));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ACrashDuringAWriteLeavesThePreviousSidecarIntact()
|
||||
{
|
||||
var path = SidecarPath();
|
||||
var first = Sample();
|
||||
ManifestStore.WriteSequence(path, first);
|
||||
// Simulated crash: the next version was half written to the temporary sibling and never renamed.
|
||||
var torn = AtomicFile.TempPathFor(path);
|
||||
File.WriteAllBytes(torn, Encoding.UTF8.GetBytes(Canonical(first with { Title = "Lost" })[..40]));
|
||||
|
||||
Assert.Equal(Canonical(first), Canonical(ManifestStore.ReadSequence(path)));
|
||||
ManifestStore.WriteSequence(path, first with { Title = "Second" });
|
||||
Assert.Equal("Second", ManifestStore.ReadSequence(path).Title);
|
||||
Assert.FileExists(torn);
|
||||
Assert.Equal(2, Directory.GetFiles(Path.GetDirectoryName(path)!).Length);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CreatingASidecarClaimsTheFolderOnce()
|
||||
{
|
||||
var path = SidecarPath();
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
var a = Sample();
|
||||
var b = Sample();
|
||||
Assert.True(ManifestStore.TryCreateSequence(path, a));
|
||||
Assert.False(ManifestStore.TryCreateSequence(path, b));
|
||||
Assert.Equal(a.Id, ManifestStore.ReadSequence(path).Id);
|
||||
Assert.Count(1, Directory.GetFiles(Path.GetDirectoryName(path)!));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ArchiveManifestRoundTripsAndKeepsUnknownFields()
|
||||
{
|
||||
var path = Path.Combine(context.CreateTempDirectory(), "archive.json");
|
||||
File.WriteAllText(path, $$"""
|
||||
{ "schemaVersion": 1, "id": "{{Guid.CreateVersion7()}}", "name": "Main", "created": "2026-10-02T09:00:00+00:00",
|
||||
"layoutTemplates": { "folder": "{year}/{title}", "file": "{n:00000}" }, "proxyPolicy": { "longEdge": 2560, "quality": 90 },
|
||||
"syncPolicy": "mirror" }
|
||||
""");
|
||||
var read = ManifestStore.ReadArchive(path);
|
||||
Assert.Equal("{year}/{title}", read.LayoutTemplates.Folder);
|
||||
Assert.Equal(2560, read.ProxyPolicy.LongEdge);
|
||||
ManifestStore.WriteArchive(path, read with { Name = "Studio" });
|
||||
Assert.Contains("syncPolicy", File.ReadAllText(path));
|
||||
Assert.Equal("Studio", ManifestStore.ReadArchive(path).Name);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
|
||||
namespace Titano.Core.Tests.Archiving;
|
||||
|
||||
/// <summary>Small real JPEGs with Exif (capture time, camera) made by the in-house encoder; each seed gives distinct bytes.</summary>
|
||||
internal static class TestPhotos
|
||||
{
|
||||
public static byte[] Jpeg(DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431",
|
||||
int width = 48, int height = 32, int orientation = 1)
|
||||
{
|
||||
using var image = new ImageBuffer(width, height);
|
||||
var random = new Random(seed);
|
||||
for (var c = 0; c < 3; c++)
|
||||
{
|
||||
var plane = image.Plane(c);
|
||||
var level = (float)random.NextDouble();
|
||||
for (var i = 0; i < plane.Length; i++) plane[i] = level * (0.5f + 0.5f * (i % width) / width);
|
||||
}
|
||||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata
|
||||
{
|
||||
Make = make,
|
||||
Model = model,
|
||||
BodySerial = serial,
|
||||
CapturedAt = captured,
|
||||
ExposureTime = 1.0 / 125,
|
||||
FNumber = 8,
|
||||
Iso = 100,
|
||||
Orientation = orientation,
|
||||
});
|
||||
return JpegEncoder.Encode(image, new JpegEncodeOptions { Quality = 85, App1Segments = [exif[4..]] });
|
||||
}
|
||||
|
||||
public static string Write(string path, DateTime captured, int seed, string make = "Canon", string model = "Canon EOS R5", string? serial = "0431")
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
File.WriteAllBytes(path, Jpeg(captured, seed, make, model, serial));
|
||||
return path;
|
||||
}
|
||||
|
||||
/// <summary>Writes <paramref name="count"/> photos named IMG_0001.JPG… taken every <paramref name="interval"/>.</summary>
|
||||
public static List<string> WriteSeries(string directory, DateTime start, int count, TimeSpan interval, int firstSeed = 1, string prefix = "IMG_",
|
||||
string make = "Canon", string model = "Canon EOS R5", int firstNumber = 1)
|
||||
{
|
||||
var paths = new List<string>(count);
|
||||
for (var i = 0; i < count; i++)
|
||||
paths.Add(Write(Path.Combine(directory, $"{prefix}{firstNumber + i:0000}.JPG"), start + interval * i, firstSeed + i, make, model));
|
||||
return paths;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Archiving;
|
||||
|
||||
[Category("archive")]
|
||||
public sealed class ThumbPackTests(TestContext context)
|
||||
{
|
||||
private static byte[] Payload(int seed, int length = 300)
|
||||
{
|
||||
var data = new byte[length];
|
||||
new Random(seed).NextBytes(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private static byte[] Hash(int seed) => ThumbPack.Fingerprint(null, $"originals/{seed}.jpg", seed);
|
||||
|
||||
private string NewPack() => Path.Combine(context.CreateTempDirectory(), ".titano", "thumbs.pack");
|
||||
|
||||
private static void Fill(string path, int count, int seedOffset = 0)
|
||||
{
|
||||
using var writer = ThumbPackWriter.Open(path);
|
||||
for (var i = 0; i < count; i++) writer.Append(i, 320, 213, Hash(i + seedOffset), Payload(i + seedOffset));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EntriesAreReadBackByFrameIndex()
|
||||
{
|
||||
var path = NewPack();
|
||||
Fill(path, 100);
|
||||
using var reader = ThumbPackReader.Open(path);
|
||||
Assert.Equal(100, reader.Entries.Count);
|
||||
Assert.BytesEqual(Payload(57), reader.Read(57));
|
||||
Assert.BytesEqual(Hash(57), reader.Entries[57].SourceHash);
|
||||
Assert.Equal(320, reader.Entries[57].Width);
|
||||
Assert.Null(reader.Read(100));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AppendingAgainReplacesAFrameAndKeepsTheOthers()
|
||||
{
|
||||
var path = NewPack();
|
||||
Fill(path, 3);
|
||||
using (var writer = ThumbPackWriter.Open(path))
|
||||
{
|
||||
Assert.Equal(3, writer.Entries.Count);
|
||||
writer.Append(1, 200, 100, Hash(99), Payload(99));
|
||||
writer.Append(3, 200, 100, Hash(3), Payload(3));
|
||||
}
|
||||
using var reader = ThumbPackReader.Open(path);
|
||||
Assert.Equal(4, reader.Entries.Count);
|
||||
Assert.BytesEqual(Payload(99), reader.Read(1));
|
||||
Assert.BytesEqual(Payload(0), reader.Read(0));
|
||||
Assert.BytesEqual(Payload(3), reader.Read(3));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnIncompleteTailIsIgnoredAndTruncatedByTheNextWriter()
|
||||
{
|
||||
var path = NewPack();
|
||||
Fill(path, 2);
|
||||
var committed = new FileInfo(path).Length;
|
||||
// Simulated crash mid-append: an entry header and part of its payload, no index.
|
||||
using (var stream = new FileStream(path, FileMode.Append))
|
||||
{
|
||||
stream.Write("TENT"u8);
|
||||
stream.Write(BitConverter.GetBytes(2));
|
||||
stream.Write(new byte[30]);
|
||||
}
|
||||
using (var reader = ThumbPackReader.Open(path))
|
||||
{
|
||||
Assert.Equal(2, reader.Entries.Count);
|
||||
Assert.BytesEqual(Payload(1), reader.Read(1));
|
||||
}
|
||||
using (var writer = ThumbPackWriter.Open(path))
|
||||
{
|
||||
Assert.Equal(committed, new FileInfo(path).Length);
|
||||
writer.Append(2, 320, 213, Hash(2), Payload(2));
|
||||
}
|
||||
using var after = ThumbPackReader.Open(path);
|
||||
Assert.Equal(3, after.Entries.Count);
|
||||
Assert.BytesEqual(Payload(2), after.Read(2));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EntriesWrittenAfterTheLastIndexAreFoundWithoutIt()
|
||||
{
|
||||
var path = NewPack();
|
||||
Fill(path, 5);
|
||||
// Simulated crash after new entries overwrote the old index but before the new index was written.
|
||||
var withoutIndex = File.ReadAllBytes(path);
|
||||
using (var reader = ThumbPackReader.Open(path))
|
||||
{
|
||||
var end = reader.Entries.Values.Max(e => e.Offset + e.Length);
|
||||
File.WriteAllBytes(path, withoutIndex[..(int)end]);
|
||||
}
|
||||
using var scanned = ThumbPackReader.Open(path);
|
||||
Assert.Equal(5, scanned.Entries.Count);
|
||||
for (var i = 0; i < 5; i++) Assert.BytesEqual(Payload(i), scanned.Read(i));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ADamagedThumbnailReadsAsMissing()
|
||||
{
|
||||
var path = NewPack();
|
||||
Fill(path, 3);
|
||||
long offset;
|
||||
using (var reader = ThumbPackReader.Open(path)) offset = reader.Entries[1].Offset;
|
||||
using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite))
|
||||
{
|
||||
stream.Position = offset + 10;
|
||||
var b = stream.ReadByte();
|
||||
stream.Position = offset + 10;
|
||||
stream.WriteByte((byte)(b ^ 0xFF));
|
||||
}
|
||||
using var damaged = ThumbPackReader.Open(path);
|
||||
Assert.Null(damaged.Read(1));
|
||||
Assert.BytesEqual(Payload(2), damaged.Read(2));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOpenReaderKeepsReadingWhileAWriterAppends()
|
||||
{
|
||||
var path = NewPack();
|
||||
Fill(path, 3);
|
||||
using var reader = ThumbPackReader.Open(path);
|
||||
using (var writer = ThumbPackWriter.Open(path))
|
||||
{
|
||||
for (var i = 3; i < 50; i++) writer.Append(i, 320, 213, Hash(i), Payload(i, 2000));
|
||||
}
|
||||
Assert.BytesEqual(Payload(0), reader.Read(0));
|
||||
Assert.BytesEqual(Payload(2), reader.Read(2));
|
||||
using var fresh = ThumbPackReader.Open(path);
|
||||
Assert.Equal(50, fresh.Entries.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void CompactionDropsReplacedEntries()
|
||||
{
|
||||
var path = NewPack();
|
||||
for (var round = 0; round < 4; round++) Fill(path, 20, seedOffset: round * 100);
|
||||
long garbage;
|
||||
using (var reader = ThumbPackReader.Open(path)) garbage = reader.GarbageBytes;
|
||||
Assert.Greater(garbage, 0);
|
||||
var before = new FileInfo(path).Length;
|
||||
ThumbPackWriter.Compact(path);
|
||||
Assert.Equal(before - garbage, new FileInfo(path).Length);
|
||||
using var compacted = ThumbPackReader.Open(path);
|
||||
Assert.Equal(0L, compacted.GarbageBytes);
|
||||
for (var i = 0; i < 20; i++) Assert.BytesEqual(Payload(i + 300), compacted.Read(i));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMissingOrForeignFileReadsAsEmpty()
|
||||
{
|
||||
var path = NewPack();
|
||||
using (var none = ThumbPackReader.Open(path)) Assert.Equal(0, none.Entries.Count);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||
File.WriteAllText(path, "not a pack");
|
||||
using (var foreign = ThumbPackReader.Open(path)) Assert.Equal(0, foreign.Entries.Count);
|
||||
Fill(path, 1);
|
||||
using var rewritten = ThumbPackReader.Open(path);
|
||||
Assert.Equal(1, rewritten.Entries.Count);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user