merge: archive layout, sidecars, scan/rebuild/verify, card import and ingest

This commit is contained in:
Claude committed 2026-10-02 12:30:53 +00:00
commit e92568a7bb
65 files changed
+7197

No files matched your search

+46
View File
@@ -0,0 +1,46 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Titano.Core.Archiving;
/// <summary>Source-generated metadata for every JSON file of the archive layout.</summary>
[JsonSourceGenerationOptions(
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
PropertyNameCaseInsensitive = true,
AllowTrailingCommas = true,
ReadCommentHandling = JsonCommentHandling.Skip)]
[JsonSerializable(typeof(ArchiveManifest))]
[JsonSerializable(typeof(SequenceManifest))]
[JsonSerializable(typeof(LeaseRecord))]
[JsonSerializable(typeof(IngestJournal))]
[JsonSerializable(typeof(RenameJournal))]
internal sealed partial class ArchiveJson : JsonSerializerContext;
/// <summary>Indented output for the small files people may open (<c>archive.json</c>, rename journals).</summary>
[JsonSourceGenerationOptions(
WriteIndented = true,
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(ArchiveManifest))]
[JsonSerializable(typeof(RenameJournal))]
internal sealed partial class ArchiveJsonIndented : JsonSerializerContext;
/// <summary>
/// Writes enum names and reads unknown names (written by a newer build) or numbers as the default value instead of
/// failing, so a sidecar from a newer Titano is still readable.
/// </summary>
public sealed class TolerantEnumConverter<TEnum> : JsonConverter<TEnum>
where TEnum : struct, Enum
{
public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.String && Enum.TryParse<TEnum>(reader.GetString(), ignoreCase: true, out var value)
&& Enum.IsDefined(value))
return value;
if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) reader.Skip();
return default;
}
public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToString());
}
+104
View File
@@ -0,0 +1,104 @@
using System.Globalization;
using Titano.Core.IO;
namespace Titano.Core.Archiving;
/// <summary>Names of the fixed files and folders of the archive layout (BRIEF §7).</summary>
public static class ArchiveLayout
{
/// <summary>Metadata folder at the archive root and inside every sequence folder.</summary>
public const string MetaFolder = ".titano";
public const string ArchiveManifestFile = "archive.json";
public const string IndexSnapshotFile = "index-snapshot.bin";
public const string LocksFolder = "locks";
/// <summary>Verified copies waiting to be committed; on the archive volume so commits are renames.</summary>
public const string StagingFolder = "staging";
/// <summary>Resumable plans of mass operations (rename).</summary>
public const string JournalFolder = "journal";
public const string OriginalsFolder = "originals";
public const string ProxiesFolder = "proxies";
public const string ExportsFolder = "exports";
public const string SequenceManifestFile = "sequence.json";
public const string AnalysisFile = "analysis.json";
public const string ProjectFile = "project.json";
public const string ThumbsPackFile = "thumbs.pack";
/// <summary>Moves of an ingest batch that were started but not yet recorded in <see cref="SequenceManifestFile"/>.</summary>
public const string IngestJournalFile = "ingest.pending.json";
/// <summary>Folder names inside a sequence folder that never hold a sequence of their own.</summary>
public static bool IsReservedFolder(string name) =>
name.Equals(MetaFolder, StringComparison.OrdinalIgnoreCase)
|| name.Equals(OriginalsFolder, StringComparison.OrdinalIgnoreCase)
|| name.Equals(ProxiesFolder, StringComparison.OrdinalIgnoreCase)
|| name.Equals(ExportsFolder, StringComparison.OrdinalIgnoreCase);
/// <summary><c>0001.jpg</c> for frame index 0: proxies are numbered from 1 like the frames of a render.</summary>
public static string ProxyFileName(int index) => (index + 1).ToString("0000", CultureInfo.InvariantCulture) + ".jpg";
}
/// <summary>Absolute paths of one archive root.</summary>
public sealed class ArchivePaths
{
public ArchivePaths(string root)
{
ArgumentException.ThrowIfNullOrWhiteSpace(root);
Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)));
MetaDirectory = Path.Combine(Root, ArchiveLayout.MetaFolder);
}
public string Root { get; }
public string MetaDirectory { get; }
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.ArchiveManifestFile);
public string SnapshotPath => Path.Combine(MetaDirectory, ArchiveLayout.IndexSnapshotFile);
public string LocksDirectory => Path.Combine(MetaDirectory, ArchiveLayout.LocksFolder);
public string StagingDirectory => Path.Combine(MetaDirectory, ArchiveLayout.StagingFolder);
public string JournalDirectory => Path.Combine(MetaDirectory, ArchiveLayout.JournalFolder);
/// <summary>Paths of the sequence stored in <paramref name="relativeFolder"/> (forward slashes).</summary>
public SequencePaths Sequence(string relativeFolder) => new(this, relativeFolder);
/// <summary>Archive-relative path with forward slashes, as stored in the index.</summary>
public string ToRelative(string path) => LongPath.ToPortableRelative(Root, path);
public string FromRelative(string relative) => LongPath.FromPortableRelative(Root, relative);
}
/// <summary>Absolute paths of one sequence folder.</summary>
public sealed class SequencePaths
{
internal SequencePaths(ArchivePaths archive, string relativeFolder)
{
ArgumentNullException.ThrowIfNull(relativeFolder);
RelativeFolder = relativeFolder.Replace('\\', '/').Trim('/');
Folder = archive.FromRelative(RelativeFolder);
MetaDirectory = Path.Combine(Folder, ArchiveLayout.MetaFolder);
}
/// <summary>Folder relative to the archive root, forward slashes; empty when the root itself holds the photos (adopted folder).</summary>
public string RelativeFolder { get; }
public string Folder { get; }
public string MetaDirectory { get; }
public string Originals => Path.Combine(Folder, ArchiveLayout.OriginalsFolder);
public string Proxies => Path.Combine(Folder, ArchiveLayout.ProxiesFolder);
public string Exports => Path.Combine(Folder, ArchiveLayout.ExportsFolder);
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.SequenceManifestFile);
public string AnalysisPath => Path.Combine(MetaDirectory, ArchiveLayout.AnalysisFile);
public string ProjectPath => Path.Combine(MetaDirectory, ArchiveLayout.ProjectFile);
public string ThumbsPackPath => Path.Combine(MetaDirectory, ArchiveLayout.ThumbsPackFile);
public string IngestJournalPath => Path.Combine(MetaDirectory, ArchiveLayout.IngestJournalFile);
public string Proxy(int index) => Path.Combine(Proxies, ArchiveLayout.ProxyFileName(index));
/// <summary>Absolute path of a frame file given its sidecar path (relative to the sequence folder).</summary>
public string File(string sequenceRelative) => LongPath.FromPortableRelative(Folder, sequenceRelative);
/// <summary>Archive-relative path of a frame file (as stored in the frames table).</summary>
public string ArchiveRelative(string sequenceRelative) =>
RelativeFolder.Length == 0 ? sequenceRelative : RelativeFolder + "/" + sequenceRelative;
}
@@ -0,0 +1,49 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Titano.Core.Naming;
namespace Titano.Core.Archiving;
/// <summary>
/// <c>.titano/archive.json</c> at the archive root (BRIEF §7). Fields this build does not know are kept in
/// <see cref="Extra"/> and written back unchanged.
/// </summary>
public sealed record ArchiveManifest
{
public const int CurrentSchemaVersion = 1;
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
public required Guid Id { get; init; }
public required string Name { get; init; }
public DateTimeOffset Created { get; init; }
public LayoutTemplates LayoutTemplates { get; init; } = new();
public ProxyPolicy ProxyPolicy { get; init; } = new();
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>Naming templates used when frames enter the archive (BRIEF §7.1).</summary>
public sealed record LayoutTemplates
{
public string Folder { get; init; } = TemplateDefaults.Folder;
public string File { get; init; } = TemplateDefaults.File;
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>Proxy generation policy (BRIEF §8.5).</summary>
public sealed record ProxyPolicy
{
public const int Disabled = -1;
public const int FullSize = 0;
/// <summary>Long edge in pixels: 1280, 1920 (default) or 2560; <see cref="FullSize"/> or <see cref="Disabled"/>.</summary>
public int LongEdge { get; init; } = 1920;
public int Quality { get; init; } = 92;
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
@@ -0,0 +1,77 @@
using Titano.Core.Imaging;
namespace Titano.Core.Archiving;
/// <summary>A folder that holds a sequence: it has a sidecar, or photos to adopt.</summary>
/// <param name="RelativeFolder">Archive-relative, forward slashes; empty for the root.</param>
/// <param name="Sidecar">Stamp of <c>.titano/sequence.json</c>, null when it has none yet.</param>
/// <param name="Photos">Stamp of the photo directory (its time changes when files are added or removed).</param>
public sealed record SequenceFolder(string RelativeFolder, FileStamp? Sidecar, FileStamp? Photos);
/// <summary>
/// Walks an archive tree (BRIEF §7). No file-system watcher is involved: callers compare the stamps with those of
/// the previous scan. Hidden folders, symbolic links and the reserved sequence subfolders are not entered.
/// </summary>
public static class ArchiveScanner
{
private const int MaxDepth = 16;
private static readonly EnumerationOptions Listing = new()
{
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System,
RecurseSubdirectories = false,
};
/// <summary>Every sequence folder under the root, ordered by relative path (ordinal) so job checkpoints are stable.</summary>
public static IReadOnlyList<SequenceFolder> FindSequenceFolders(ArchivePaths archive, CancellationToken cancellationToken = default)
{
var found = new List<SequenceFolder>();
var pending = new Stack<(string Path, int Depth)>();
pending.Push((archive.Root, 0));
while (pending.Count > 0)
{
cancellationToken.ThrowIfCancellationRequested();
var (path, depth) = pending.Pop();
var relative = archive.ToRelative(path);
var sequence = archive.Sequence(relative == "." ? string.Empty : relative);
var sidecar = FileStamp.Of(sequence.ManifestPath);
var photoDirectory = PhotoDirectory(sequence);
if (sidecar is not null || EnumeratePhotos(photoDirectory).Any())
found.Add(new SequenceFolder(sequence.RelativeFolder, sidecar, FileStamp.Of(photoDirectory)));
if (depth >= MaxDepth) continue;
foreach (var child in Directory.EnumerateDirectories(path, "*", Listing))
{
var name = Path.GetFileName(child);
if (!IsHidden(child, name) && !ArchiveLayout.IsReservedFolder(name)) pending.Push((child, depth + 1));
}
}
found.Sort((a, b) => string.CompareOrdinal(a.RelativeFolder, b.RelativeFolder));
return found;
}
/// <summary><c>originals/</c> when the folder follows the layout, otherwise the folder itself (adopted in place).</summary>
public static string PhotoDirectory(SequencePaths sequence) => Directory.Exists(sequence.Originals) ? sequence.Originals : sequence.Folder;
/// <summary>Photos of a sequence folder as sidecar paths (relative to the sequence folder), ordinal order.</summary>
public static IReadOnlyList<string> ListPhotos(SequencePaths sequence)
{
var directory = PhotoDirectory(sequence);
return EnumeratePhotos(directory)
.Select(path => Path.GetRelativePath(sequence.Folder, path).Replace('\\', '/'))
.Order(StringComparer.Ordinal)
.ToList();
}
/// <summary>True for files Titano imports as photos (AppleDouble <c>._</c> companions excluded).</summary>
public static bool IsPhoto(string fileName) =>
!fileName.StartsWith("._", StringComparison.Ordinal) && ImageFormatInfo.IsSupportedExtension(fileName);
private static IEnumerable<string> EnumeratePhotos(string directory) =>
Directory.Exists(directory)
? Directory.EnumerateFiles(directory, "*", Listing).Where(f => IsPhoto(Path.GetFileName(f)) && !IsHidden(f, Path.GetFileName(f)))
: [];
private static bool IsHidden(string path, string name) =>
name.StartsWith('.') || (OperatingSystem.IsWindows() && File.GetAttributes(path).HasFlag(FileAttributes.Hidden));
}
@@ -0,0 +1,55 @@
using Titano.Core.Localization;
namespace Titano.Core.Archiving;
/// <summary>The archive cannot be reached (or written) right now; nothing was changed.</summary>
public sealed class ArchiveOfflineException(string root, ArchiveAvailability availability)
: IOException(Loc.L(availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", root))
{
public string Root { get; } = root;
public ArchiveAvailability Availability { get; } = availability;
}
public enum ArchiveAvailability
{
Online,
/// <summary>Readable but not writable (read-only share, write-protected disk): browse only.</summary>
ReadOnly,
/// <summary>Root unreachable (unplugged drive, disconnected share) or no longer an archive.</summary>
Offline,
}
/// <summary>Offline detection for archives on removable and network volumes (BRIEF §7).</summary>
public static class ArchiveVolume
{
/// <summary>Throws <see cref="ArchiveOfflineException"/> unless the archive is online (writable).</summary>
public static void EnsureWritable(ArchivePaths archive)
{
var availability = Probe(archive);
if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(archive.Root, availability);
}
/// <summary>Checks that <c>archive.json</c> is reachable and that the metadata folder accepts writes.</summary>
public static ArchiveAvailability Probe(ArchivePaths archive)
{
try
{
if (!File.Exists(archive.ManifestPath)) return ArchiveAvailability.Offline;
var probe = Path.Combine(archive.MetaDirectory, ".probe-" + Guid.NewGuid().ToString("N")[..8]);
using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1, FileOptions.DeleteOnClose))
{
}
return ArchiveAvailability.Online;
}
catch (UnauthorizedAccessException)
{
return ArchiveAvailability.ReadOnly;
}
catch (IOException)
{
return File.Exists(archive.ManifestPath) ? ArchiveAvailability.ReadOnly : ArchiveAvailability.Offline;
}
}
}
+44
View File
@@ -0,0 +1,44 @@
using System.Security.Cryptography;
namespace Titano.Core.Archiving;
/// <summary>SHA-256 of files, streamed (originals can be hundreds of megabytes).</summary>
public static class ContentHash
{
internal const int BufferSize = 1 << 20;
/// <summary>Lower-case hex SHA-256 of the file.</summary>
/// <param name="progress">Receives the number of bytes read since the previous call.</param>
public static async Task<string> Sha256Async(string path, Action<long>? progress = null, CancellationToken cancellationToken = default)
{
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, BufferSize,
FileOptions.Asynchronous | FileOptions.SequentialScan);
return await Sha256Async(stream, progress, cancellationToken).ConfigureAwait(false);
}
public static async Task<string> Sha256Async(Stream stream, Action<long>? progress = null, CancellationToken cancellationToken = default)
{
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var buffer = new byte[BufferSize];
int read;
while ((read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
{
hash.AppendData(buffer, 0, read);
progress?.Invoke(read);
}
return Convert.ToHexStringLower(hash.GetHashAndReset());
}
}
/// <summary>Size and modification time of a file, the cheap change detector of incremental scans (BRIEF §7).</summary>
public readonly record struct FileStamp(long Size, DateTime ModifiedUtc)
{
/// <summary>The stamp of a file or directory, or null when it does not exist.</summary>
public static FileStamp? Of(string path)
{
var file = new FileInfo(path);
if (file.Exists) return new FileStamp(file.Length, file.LastWriteTimeUtc);
var directory = new DirectoryInfo(path);
return directory.Exists ? new FileStamp(0, directory.LastWriteTimeUtc) : null;
}
}
@@ -0,0 +1,97 @@
using Titano.Core.Codecs.Jpeg;
using Titano.Core.Imaging;
using Titano.Core.Metadata;
namespace Titano.Core.Archiving;
/// <summary>A JPEG made from a frame.</summary>
public sealed record EncodedImage(byte[] Jpeg, int Width, int Height, DecodeTier Tier);
/// <summary>
/// Makes thumbnails and proxies of frames (BRIEF §8.5): the embedded preview or the managed JPEG decoder first, then
/// the host's decode providers (WIC on Windows); orientation applied, resampled in linear light, encoded in-house.
/// </summary>
public sealed class FrameImageRenderer(ImageDecoder decoder)
{
public const int ThumbnailLongEdge = 320;
public const int ThumbnailQuality = 80;
/// <summary>A 320 px q80 thumbnail, or null when no decoder can read the frame.</summary>
public EncodedImage? Thumbnail(SequencePaths sequence, SidecarFrame frame, CancellationToken cancellationToken = default)
{
using var decoded = Decode(sequence.File(frame.File), frame, ThumbnailLongEdge, DecodePurpose.Thumbnail, preferPreview: true, cancellationToken);
if (decoded is null) return null;
using var small = Resampler.Thumbnail(decoded.Pixels, ThumbnailLongEdge);
return Encode(small, new JpegEncodeOptions { Quality = ThumbnailQuality }, decoded.Tier);
}
/// <summary>A proxy at the policy's long edge and quality with capture time and camera in its Exif; null when undecodable.</summary>
public EncodedImage? Proxy(SequencePaths sequence, SidecarFrame frame, SidecarCamera? camera, ProxyPolicy policy, CancellationToken cancellationToken = default)
{
if (policy.LongEdge == ProxyPolicy.Disabled) throw new InvalidOperationException("Proxies are disabled for this archive.");
using var decoded = Decode(sequence.File(frame.File), frame, policy.LongEdge, DecodePurpose.Proxy, preferPreview: false, cancellationToken);
if (decoded is null) return null;
var source = decoded.Pixels;
var (w, h) = policy.LongEdge == ProxyPolicy.FullSize ? (source.Width, source.Height) : Resampler.FitWithin(source.Width, source.Height, policy.LongEdge);
using var resized = Resampler.Resize(source, w, h);
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata
{
Make = camera?.Make,
Model = camera?.Model,
BodySerial = camera?.Serial,
CapturedAt = frame.CapturedAt,
UtcOffsetMinutes = frame.UtcOffset,
ExposureTime = frame.Exposure,
FNumber = frame.FNumber,
Iso = frame.Iso,
FocalLength = frame.Focal,
ExposureBias = frame.Bias,
Software = "Titano",
});
// The encoder takes the APP1 payload; ExifWriter returns the whole segment (marker and length first).
return Encode(resized, new JpegEncodeOptions { Quality = policy.Quality, App1Segments = [exif[4..]] }, decoded.Tier);
}
private static EncodedImage Encode(ImageBuffer pixels, JpegEncodeOptions options, DecodeTier tier) =>
new(JpegEncoder.Encode(pixels, options), pixels.Width, pixels.Height, tier);
private DecodedFrame? Decode(string path, SidecarFrame frame, int maxLongEdge, DecodePurpose purpose, bool preferPreview, CancellationToken cancellationToken)
{
if (frame.Format == ImageFileFormat.Jpeg) return DecodeJpeg(File.ReadAllBytes(path), maxLongEdge, frame.Orientation, DecodeTier.Native);
if (preferPreview && ReadPreview(path, frame) is { } preview) return DecodeJpeg(preview, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview);
var request = new DecodeRequest(path, System.Math.Max(0, maxLongEdge), purpose, ApplyOrientation: true, AllowPreview: true);
if (decoder.Decode(request, cancellationToken) is { } decoded) return new DecodedFrame(decoded.Pixels, decoded.Tier);
return ReadPreview(path, frame) is { } fallback ? DecodeJpeg(fallback, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview) : null;
}
private static DecodedFrame? DecodeJpeg(byte[] data, int maxLongEdge, int orientation, DecodeTier tier)
{
ImageBuffer decoded;
try
{
decoded = JpegDecoder.DecodeToLinear(data, new JpegDecodeOptions { MaxLongEdge = System.Math.Max(0, maxLongEdge) }, expandGray: true);
}
catch (InvalidDataException)
{
return null;
}
if (orientation is <= 1 or > 8) return new DecodedFrame(decoded, tier);
using (decoded) return new DecodedFrame(PixelConvert.ApplyOrientation(decoded, orientation), tier);
}
private static byte[]? ReadPreview(string path, SidecarFrame frame)
{
if (frame.PreviewOffset is not { } offset || frame.PreviewLength is not ({ } length and > 0 and < int.MaxValue)) return null;
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 4096, FileOptions.RandomAccess);
if (offset < 0 || offset + length > stream.Length) return null;
var data = new byte[length];
stream.Position = offset;
stream.ReadExactly(data);
return data;
}
private sealed record DecodedFrame(ImageBuffer Pixels, DecodeTier Tier) : IDisposable
{
public void Dispose() => Pixels.Dispose();
}
}
@@ -0,0 +1,54 @@
using Titano.Core.Model;
namespace Titano.Core.Archiving;
public enum FrameHealth
{
/// <summary>Same size and SHA-256 as recorded.</summary>
Ok,
/// <summary>No hash was recorded (adopted file); it has been computed now.</summary>
Hashed,
Missing,
/// <summary>A different size: the file was replaced or edited.</summary>
Changed,
/// <summary>Same size, different content: bit rot or a damaged copy.</summary>
Corrupt,
}
/// <param name="Sha256">The hash computed now (null when the file is missing or changed size).</param>
public sealed record FrameCheck(SidecarFrame Frame, FrameHealth Health, string? Sha256)
{
/// <summary>Frame flags after this check: <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> set or cleared.</summary>
public FrameFlags Flags => Health switch
{
FrameHealth.Missing => (Frame.Flags & ~FrameFlags.Corrupt) | FrameFlags.Missing,
FrameHealth.Changed or FrameHealth.Corrupt => (Frame.Flags & ~FrameFlags.Missing) | FrameFlags.Corrupt,
_ => Frame.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt),
};
/// <summary>The sidecar record after this check (flags, and the hash of a newly hashed file).</summary>
public SidecarFrame Updated => Frame with { Flags = Flags, Sha256 = Health == FrameHealth.Hashed ? Sha256 : Frame.Sha256 };
public bool ChangesRecord => Updated != Frame;
}
/// <summary>Re-hashes originals against their sidecar records (BRIEF §12.5 Verify).</summary>
public static class FrameVerifier
{
public static async Task<FrameCheck> CheckAsync(SequencePaths sequence, SidecarFrame frame, Action<long>? bytesRead = null,
CancellationToken cancellationToken = default)
{
var info = new FileInfo(sequence.File(frame.File));
if (!info.Exists) return new FrameCheck(frame, FrameHealth.Missing, null);
if (info.Length != frame.Size) return new FrameCheck(frame, FrameHealth.Changed, null);
var sha = await ContentHash.Sha256Async(info.FullName, bytesRead, cancellationToken).ConfigureAwait(false);
var health = frame.Sha256 is null ? FrameHealth.Hashed
: sha.Equals(frame.Sha256, StringComparison.OrdinalIgnoreCase) ? FrameHealth.Ok
: FrameHealth.Corrupt;
return new FrameCheck(frame, health, sha);
}
}
+127
View File
@@ -0,0 +1,127 @@
using System.Buffers.Binary;
using System.Security.Cryptography;
using System.Text;
using Titano.Core.IO;
namespace Titano.Core.Archiving;
/// <summary>
/// <c>.titano/index-snapshot.bin</c> (BRIEF §7): every sidecar of the archive in one file, so a rebuild reads one file
/// sequentially instead of thousands of small ones over a share. Optional and never authoritative: an entry is used
/// only when its sidecar still has the recorded size and time, and a file whose trailing SHA-256 does not match is
/// ignored entirely.
/// </summary>
/// <remarks>Layout: "TISN", version (u16), reserved (u16), then per sidecar: folder length (u16), UTF-8 folder,
/// size (i64), modified UTC ticks (i64), length (i32), sidecar bytes; finally the SHA-256 of everything before it.</remarks>
public sealed class IndexSnapshot
{
private const ushort Version = 1;
private static ReadOnlySpan<byte> Magic => "TISN"u8;
private readonly Dictionary<string, (FileStamp Stamp, byte[] Bytes)> _entries;
private IndexSnapshot(Dictionary<string, (FileStamp, byte[])> entries) => _entries = entries;
public int Count => _entries.Count;
/// <summary>The snapshot at <paramref name="path"/>, or null when it is absent, damaged or of another version.</summary>
public static IndexSnapshot? TryLoad(string path)
{
byte[] data;
try
{
if (!File.Exists(path)) return null;
data = File.ReadAllBytes(path);
}
catch (IOException)
{
return null;
}
if (data.Length < 8 + 32 || !data.AsSpan(0, 4).SequenceEqual(Magic) || BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan(4)) != Version)
return null;
var body = data.AsSpan(0, data.Length - 32);
if (!SHA256.HashData(body).AsSpan().SequenceEqual(data.AsSpan(data.Length - 32))) return null;
var entries = new Dictionary<string, (FileStamp, byte[])>(StringComparer.Ordinal);
var position = 8;
while (position < body.Length)
{
if (position + 2 > body.Length) return null;
var folderLength = BinaryPrimitives.ReadUInt16LittleEndian(body[position..]);
position += 2;
if (position + folderLength + 20 > body.Length) return null;
var folder = Encoding.UTF8.GetString(body.Slice(position, folderLength));
position += folderLength;
var size = BinaryPrimitives.ReadInt64LittleEndian(body[position..]);
var ticks = BinaryPrimitives.ReadInt64LittleEndian(body[(position + 8)..]);
var length = BinaryPrimitives.ReadInt32LittleEndian(body[(position + 16)..]);
position += 20;
if (length < 0 || ticks < 0 || ticks > DateTime.MaxValue.Ticks || position + length > body.Length) return null;
entries[folder] = (new FileStamp(size, new DateTime(ticks, DateTimeKind.Utc)), body.Slice(position, length).ToArray());
position += length;
}
return new IndexSnapshot(entries);
}
/// <summary>The recorded sidecar bytes of <paramref name="folder"/> when its current stamp still matches.</summary>
public byte[]? TryGet(string folder, FileStamp current) =>
_entries.TryGetValue(folder, out var entry) && entry.Stamp == current ? entry.Bytes : null;
/// <summary>Streams a new snapshot to a temporary file; <see cref="Commit"/> replaces the old one atomically.</summary>
public sealed class Writer : IDisposable
{
private readonly string _path;
private readonly string _temp;
private readonly FileStream _stream;
private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
private bool _committed;
public Writer(string path)
{
_path = path;
_temp = AtomicFile.TempPathFor(path);
_stream = new FileStream(_temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16);
Span<byte> header = stackalloc byte[8];
header.Clear();
Magic.CopyTo(header);
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
Write(header);
}
public void Add(string folder, FileStamp stamp, ReadOnlySpan<byte> sidecar)
{
var name = Encoding.UTF8.GetBytes(folder);
Span<byte> head = stackalloc byte[2];
BinaryPrimitives.WriteUInt16LittleEndian(head, checked((ushort)name.Length));
Write(head);
Write(name);
Span<byte> fields = stackalloc byte[20];
BinaryPrimitives.WriteInt64LittleEndian(fields, stamp.Size);
BinaryPrimitives.WriteInt64LittleEndian(fields[8..], stamp.ModifiedUtc.Ticks);
BinaryPrimitives.WriteInt32LittleEndian(fields[16..], sidecar.Length);
Write(fields);
Write(sidecar);
}
public void Commit()
{
_stream.Write(_hash.GetHashAndReset());
_stream.Dispose();
AtomicFile.Commit(_temp, _path);
_committed = true;
}
public void Dispose()
{
_stream.Dispose();
_hash.Dispose();
if (!_committed && File.Exists(_temp)) File.Delete(_temp);
}
private void Write(ReadOnlySpan<byte> data)
{
_hash.AppendData(data);
_stream.Write(data);
}
}
}
@@ -0,0 +1,52 @@
using System.Text.Json;
using Titano.Core.IO;
namespace Titano.Core.Archiving;
/// <summary>
/// Write-ahead record of an ingest batch (<c>.titano/ingest.pending.json</c>): the frames whose files are about to be
/// moved into the sequence folder. Written before the moves and removed once <c>sequence.json</c> lists them, so a
/// crash in between is repaired by <see cref="Recover"/> instead of leaving unindexed originals behind.
/// </summary>
public sealed record IngestJournal(IReadOnlyList<SidecarFrame> Frames)
{
public static void Write(SequencePaths sequence, IngestJournal journal)
{
Directory.CreateDirectory(sequence.MetaDirectory);
AtomicFile.WriteAllBytes(sequence.IngestJournalPath, JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJson.Default.IngestJournal));
}
public static void Delete(SequencePaths sequence)
{
if (File.Exists(sequence.IngestJournalPath)) File.Delete(sequence.IngestJournalPath);
}
/// <summary>
/// Completes an interrupted batch: frames whose file reached the sequence folder (same size) but are missing from
/// <paramref name="manifest"/> are added in capture order. Returns the manifest to write, or null when nothing has
/// to change. The journal is left in place; <see cref="Delete"/> it once the result is written.
/// </summary>
public static SequenceManifest? Recover(SequencePaths sequence, SequenceManifest manifest)
{
if (!File.Exists(sequence.IngestJournalPath)) return null;
IngestJournal? journal;
try
{
journal = JsonSerializer.Deserialize(File.ReadAllBytes(sequence.IngestJournalPath), ArchiveJson.Default.IngestJournal);
}
catch (JsonException)
{
journal = null; // torn while being written: nothing was moved yet
}
if (journal is null) return null;
var known = manifest.Frames.Select(f => f.Id).ToHashSet();
var moved = journal.Frames.Where(f => !known.Contains(f.Id) && FileHasSize(sequence.File(f.File), f.Size)).ToList();
return moved.Count == 0 ? null : SequenceFrames.Insert(manifest, moved);
}
private static bool FileHasSize(string path, long size)
{
var info = new FileInfo(path);
return info.Exists && info.Length == size;
}
}
+241
View File
@@ -0,0 +1,241 @@
using System.Text.Json;
using Titano.Core.Diagnostics;
using Titano.Core.Localization;
namespace Titano.Core.Archiving;
/// <summary>Content of a lease file: who holds it and until when.</summary>
public sealed record LeaseRecord(string Token, string Owner, string Host, int ProcessId, DateTimeOffset Acquired, DateTimeOffset Expires);
/// <summary>The lease is held by someone else and did not free up in time.</summary>
public sealed class LeaseUnavailableException(string name, LeaseRecord? holder)
: IOException(Loc.L("Archive.Error.Busy", holder?.Owner ?? "?"))
{
public string Name { get; } = name;
public LeaseRecord? Holder { get; } = holder;
}
/// <summary>
/// Cross-process exclusive lease stored as <c>.titano/locks/&lt;name&gt;.lease</c> (BRIEF §7), for writers that share
/// an archive over a network (server and client). Creation uses create-new semantics, which SMB and local file
/// systems both make atomic. A lease past its expiry is stale (its holder crashed or lost the share) and is taken
/// over by renaming it aside, which only one contender can do. The holder renews it in the background while held.
/// </summary>
public sealed class LeaseFile : IDisposable
{
public static readonly TimeSpan DefaultDuration = TimeSpan.FromMinutes(2);
private const string Extension = ".lease";
private const string StaleExtension = ".stale";
private readonly Lock _gate = new();
private readonly Timer _renewal;
private LeaseRecord _record;
private bool _released;
private LeaseFile(string path, string name, LeaseRecord record, TimeSpan duration)
{
Path = path;
Name = name;
_record = record;
Duration = duration;
var period = duration / 3;
_renewal = new Timer(_ => Renew(), null, period, period);
}
public string Path { get; }
public string Name { get; }
public TimeSpan Duration { get; }
public LeaseRecord Record => _record;
/// <summary>False once a renewal found the lease taken over (it expired while this process was stalled).</summary>
public bool IsHeld { get; private set; } = true;
/// <summary>Takes the lease when it is free or stale; null when someone else holds it.</summary>
public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null)
{
var length = duration ?? DefaultDuration;
var path = PathFor(locksDirectory, name);
Directory.CreateDirectory(locksDirectory);
for (var attempt = 0; attempt < 3; attempt++)
{
var now = DateTimeOffset.UtcNow;
var record = new LeaseRecord(Guid.NewGuid().ToString("N"), owner, Environment.MachineName, Environment.ProcessId, now, now + length);
if (TryCreate(path, record)) return new LeaseFile(path, name, record, length);
if (!IsStale(path, now)) return null;
BreakStale(path, now);
}
return null;
}
/// <summary>Waits up to <paramref name="wait"/> for the lease, polling with back-off.</summary>
/// <exception cref="LeaseUnavailableException">Still held by someone else after <paramref name="wait"/>.</exception>
public static async Task<LeaseFile> AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait,
TimeSpan? duration = null, CancellationToken cancellationToken = default)
{
var deadline = DateTime.UtcNow + wait;
var delay = TimeSpan.FromMilliseconds(20);
while (true)
{
if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease;
if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name)));
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, 500));
}
}
/// <summary>The current holder of a lease, or null when free or unreadable.</summary>
public static LeaseRecord? Read(string path)
{
try
{
return JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.LeaseRecord);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
{
return null;
}
}
/// <summary>Removes every stale lease in <paramref name="locksDirectory"/> (maintenance); returns their names.</summary>
public static IReadOnlyList<string> CleanupStale(string locksDirectory, bool dryRun)
{
if (!Directory.Exists(locksDirectory)) return [];
var now = DateTimeOffset.UtcNow;
var removed = new List<string>();
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + Extension))
{
if (!IsStale(path, now)) continue;
removed.Add(System.IO.Path.GetFileNameWithoutExtension(path));
if (!dryRun) BreakStale(path, now);
}
if (!dryRun)
{
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + StaleExtension))
{
if (File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime) TryDelete(path);
}
}
return removed;
}
/// <summary>Extends the expiry; false when the lease is no longer ours.</summary>
public bool Renew()
{
lock (_gate)
{
if (_released || !IsHeld) return false;
if (Read(Path)?.Token != _record.Token)
{
IsHeld = false;
Log.Warning("archive", $"Lease {Name} was taken over by another writer");
return false;
}
var renewed = _record with { Expires = DateTimeOffset.UtcNow + Duration };
try
{
// In-place rewrite: a rename would open a window in which the lease looks free.
using var fs = new FileStream(Path, FileMode.Truncate, FileAccess.Write, FileShare.Read);
fs.Write(JsonSerializer.SerializeToUtf8Bytes(renewed, ArchiveJson.Default.LeaseRecord));
fs.Flush(flushToDisk: true);
_record = renewed;
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Runs on a timer thread: an escaping exception would end the process.
Log.Warning("archive", $"Lease {Name} could not be renewed", ex);
return false;
}
}
}
public void Dispose()
{
_renewal.Dispose();
lock (_gate)
{
if (_released) return;
_released = true;
if (!IsHeld || Read(Path)?.Token != _record.Token) return;
try
{
File.Delete(Path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// It expires on its own.
Log.Warning("archive", $"Lease {Name} could not be released", ex);
}
}
}
internal static string PathFor(string locksDirectory, string name) => System.IO.Path.Combine(locksDirectory, name + Extension);
private static bool TryCreate(string path, LeaseRecord record)
{
try
{
using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
fs.Flush(flushToDisk: true);
return true;
}
catch (IOException) when (File.Exists(path))
{
return false;
}
}
/// <summary>Expired, or unreadable for longer than a lease lasts (a holder that crashed while creating it).</summary>
private static bool IsStale(string path, DateTimeOffset now)
{
if (Read(path) is { } record) return record.Expires < now;
try
{
return File.Exists(path) && File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime;
}
catch (IOException)
{
return false;
}
}
private static void BreakStale(string path, DateTimeOffset now)
{
var aside = path + "." + Guid.NewGuid().ToString("N")[..8] + StaleExtension;
try
{
File.Move(path, aside);
}
catch (IOException)
{
return; // another contender broke it first
}
if (IsStale(aside, now))
{
TryDelete(aside);
return;
}
// A contender renewed or re-created the lease between our check and the rename: give it back.
try
{
File.Move(aside, path, overwrite: false);
}
catch (IOException)
{
TryDelete(aside);
}
}
private static void TryDelete(string path)
{
try
{
File.Delete(path);
}
catch (IOException)
{
// A leftover file is harmless; maintenance removes it.
}
}
}
@@ -0,0 +1,84 @@
using System.Text.Json;
using Titano.Core.IO;
using Titano.Core.Localization;
namespace Titano.Core.Archiving;
/// <summary>A sidecar written by a newer Titano: it can be read, but this build refuses to rewrite it.</summary>
public sealed class NewerFormatException(string path, int version)
: InvalidOperationException(Loc.L("Archive.Error.NewerFormat", System.IO.Path.GetFileName(path), version))
{
public string Path { get; } = path;
public int Version { get; } = version;
}
/// <summary>
/// Reads and writes the JSON files of the layout. Reading tolerates unknown fields and enum values; every write
/// goes through <see cref="AtomicFile"/> (temp → fsync → rename), so a crash leaves the previous file intact.
/// </summary>
public static class ManifestStore
{
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
public static ArchiveManifest ReadArchive(string path) => Parse(File.ReadAllBytes(path), path, ArchiveJson.Default.ArchiveManifest);
public static void WriteArchive(string path, ArchiveManifest manifest)
{
RefuseNewer(path, manifest.SchemaVersion, ArchiveManifest.CurrentSchemaVersion);
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJsonIndented.Default.ArchiveManifest));
}
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
public static SequenceManifest ReadSequence(string path) => ParseSequence(File.ReadAllBytes(path), path);
/// <summary>Parses sidecar bytes (from disk or from the index snapshot).</summary>
public static SequenceManifest ParseSequence(ReadOnlySpan<byte> utf8, string source) => Parse(utf8, source, ArchiveJson.Default.SequenceManifest);
/// <summary>Replaces the sidecar atomically. Compact JSON: a 100k-frame sidecar stays around 35 MB.</summary>
public static void WriteSequence(string path, SequenceManifest manifest)
{
RefuseNewer(path, manifest.SchemaVersion, SequenceManifest.CurrentSchemaVersion);
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
}
/// <summary>
/// Creates the sidecar only if none exists (rename without replace), which makes it the claim on a new
/// sequence folder between processes sharing the archive. False when another sidecar is already there.
/// </summary>
public static bool TryCreateSequence(string path, SequenceManifest manifest)
{
if (File.Exists(path)) return false;
var temp = AtomicFile.TempPathFor(path);
File.WriteAllBytes(temp, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
try
{
using (var fs = new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) fs.Flush(flushToDisk: true);
File.Move(temp, path, overwrite: false);
return true;
}
catch (IOException) when (File.Exists(path))
{
return false;
}
finally
{
if (File.Exists(temp)) File.Delete(temp);
}
}
private static T Parse<T>(ReadOnlySpan<byte> utf8, string source, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo)
{
try
{
return JsonSerializer.Deserialize(utf8, typeInfo) ?? throw new InvalidDataException($"{source} is empty.");
}
catch (JsonException ex)
{
throw new InvalidDataException($"{source} is not a valid Titano file: {ex.Message}", ex);
}
}
private static void RefuseNewer(string path, int version, int supported)
{
if (version > supported) throw new NewerFormatException(path, version);
}
}
+131
View File
@@ -0,0 +1,131 @@
using System.Globalization;
using System.Text.Json;
using Titano.Core.IO;
using Titano.Core.Naming;
namespace Titano.Core.Archiving;
/// <summary>What the planner needs to know about a sequence to rename it.</summary>
public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens);
/// <summary>One sequence's change: its folder, its title, or both.</summary>
public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle)
{
public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal);
}
/// <summary>The exact changes a mass rename will make (its dry run); applying it makes exactly these.</summary>
public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList<RenameStep> Steps);
/// <summary>A plan being applied, with the steps already completed; lets an interrupted rename resume (BRIEF §7).</summary>
public sealed record RenameJournal(RenamePlan Plan, IReadOnlyList<Guid> Completed)
{
public static string PathFor(ArchivePaths archive, Guid planId) => Path.Combine(archive.JournalDirectory, $"rename-{planId:N}.json");
public static void Write(ArchivePaths archive, RenameJournal journal)
{
Directory.CreateDirectory(archive.JournalDirectory);
AtomicFile.WriteAllBytes(PathFor(archive, journal.Plan.Id), JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJsonIndented.Default.RenameJournal));
}
/// <summary>Journals left by interrupted renames.</summary>
public static IReadOnlyList<RenameJournal> ListPending(ArchivePaths archive)
{
if (!Directory.Exists(archive.JournalDirectory)) return [];
var list = new List<RenameJournal>();
foreach (var path in Directory.EnumerateFiles(archive.JournalDirectory, "rename-*.json").Order(StringComparer.Ordinal))
{
if (JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.RenameJournal) is { } journal) list.Add(journal);
}
return list;
}
public static void Delete(ArchivePaths archive, Guid planId) => File.Delete(PathFor(archive, planId));
}
/// <summary>Builds rename plans from folder/title templates (BRIEF §7.1, §12.5).</summary>
public static class RenamePlanner
{
/// <summary>
/// Expands <paramref name="titleTemplate"/> (where <c>{title}</c> is the current title) and then
/// <paramref name="folderTemplate"/> (where <c>{title}</c> is the new title) for every subject. A target folder
/// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"….
/// Null templates leave that part unchanged; subjects without a change produce no step.
/// </summary>
/// <param name="occupied">Every sequence folder of the archive (renamed or not).</param>
public static RenamePlan Plan(Guid archiveId, IReadOnlyList<RenameSubject> subjects, string? folderTemplate, string? titleTemplate,
IEnumerable<string> occupied, Func<string, bool> existsOnDisk)
{
var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate);
var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate);
var claimed = new HashSet<string>(occupied, StringComparer.OrdinalIgnoreCase);
var steps = new List<RenameStep>();
foreach (var s in subjects)
{
var context = new NamingContext
{
CapturedAt = s.FirstCapture ?? DateTime.Today,
SessionStart = s.FirstCapture ?? DateTime.Today,
CameraMake = s.CameraMake,
CameraModel = s.CameraModel,
Lens = s.Lens,
Title = s.Title,
};
var newTitle = title is null ? s.Title : Blank(title.ExpandFile(context)) ?? s.Title;
var newFolder = s.Folder;
// The archive root itself (a folder adopted in place) cannot move; only its title changes.
if (folder is not null && s.Folder.Length > 0 && Blank(folder.ExpandFolder(context with { Title = newTitle })) is { } expanded
&& !expanded.Equals(s.Folder, StringComparison.Ordinal))
{
newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk);
claimed.Add(newFolder);
}
if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle));
}
return new RenamePlan(Guid.CreateVersion7(), archiveId, steps);
}
/// <summary>
/// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted,
/// a case-only change goes through a temporary name (case-insensitive file systems).
/// </summary>
/// <exception cref="IOException">The target is occupied by something else, or neither folder exists.</exception>
public static void MoveFolder(ArchivePaths archive, RenameStep step)
{
if (!step.MovesFolder) return;
var from = archive.FromRelative(step.FromFolder);
var to = archive.FromRelative(step.ToFolder);
var caseOnly = step.FromFolder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase);
if (Directory.Exists(to) && (!caseOnly || HasExactName(to)))
{
if (Directory.Exists(from) && !caseOnly) throw new IOException($"Rename target {step.ToFolder} already exists.");
return;
}
if (!Directory.Exists(from)) throw new DirectoryNotFoundException($"Sequence folder {step.FromFolder} is missing.");
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
if (caseOnly)
{
var temp = from + ".rename-" + Guid.NewGuid().ToString("N")[..8];
Directory.Move(from, temp);
Directory.Move(temp, to);
}
else
{
Directory.Move(from, to);
}
}
private static bool HasExactName(string path) =>
Directory.EnumerateDirectories(Path.GetDirectoryName(path)!, Path.GetFileName(path))
.Any(d => Path.GetFileName(d).Equals(Path.GetFileName(path), StringComparison.Ordinal));
private static string? Blank(string value) => string.IsNullOrWhiteSpace(value) ? null : value;
private static string Unique(string folder, HashSet<string> claimed, Func<string, bool> existsOnDisk)
{
var candidate = folder;
for (var n = 2; claimed.Contains(candidate) || existsOnDisk(candidate); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})");
return candidate;
}
}
@@ -0,0 +1,91 @@
using Titano.Core.Diagnostics;
using Titano.Core.Metadata;
using Titano.Core.Model;
namespace Titano.Core.Archiving;
/// <summary>Files of a sequence folder that the sidecar does not list, and sidecar frames whose file is gone.</summary>
public sealed record SequenceOrphans(string Folder, IReadOnlyList<string> UnindexedFiles, IReadOnlyList<SidecarFrame> MissingFiles)
{
public bool IsEmpty => UnindexedFiles.Count == 0 && MissingFiles.Count == 0;
}
/// <summary>
/// Builds and reconciles sidecars from the photos found in a folder (BRIEF §7, §12.5 "Adopt folder"). Only light
/// metadata is read; originals are never moved, renamed or rewritten. Hashes are left empty for a verify to fill.
/// </summary>
public static class SequenceAdopter
{
/// <summary>A new sidecar for a folder without one. A folder outside the layout (no <c>originals/</c>) is titled by its name.</summary>
public static SequenceManifest Build(Guid id, SequencePaths sequence, string? createdBy, CancellationToken cancellationToken = default)
{
var (frames, first) = ReadFrames(sequence, ArchiveScanner.ListPhotos(sequence), cancellationToken);
var now = DateTimeOffset.UtcNow;
var inLayout = Directory.Exists(sequence.Originals);
var manifest = new SequenceManifest
{
Id = id,
Title = inLayout ? null : Path.GetFileName(Path.TrimEndingDirectorySeparator(sequence.Folder)),
Camera = first is null ? null : new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial },
Lens = first?.LensModel,
CreatedBy = createdBy,
Created = now,
UpdatedAt = now,
};
return SequenceFrames.Insert(manifest, frames);
}
/// <summary>Unindexed photos and missing frame files of one sequence.</summary>
public static SequenceOrphans FindOrphans(SequencePaths sequence, SequenceManifest manifest)
{
var listed = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal);
var unindexed = ArchiveScanner.ListPhotos(sequence).Where(f => !listed.Contains(f)).ToList();
var missing = manifest.Frames.Where(f => !File.Exists(sequence.File(f.File))).ToList();
return new SequenceOrphans(sequence.RelativeFolder, unindexed, missing);
}
/// <summary>
/// Brings a sidecar in line with its folder: unindexed photos are added (in capture order), frames whose file is
/// gone get <see cref="FrameFlags.Missing"/> and frames whose file came back lose it. Null when nothing changed.
/// </summary>
public static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken = default)
{
var orphans = FindOrphans(sequence, manifest);
var missing = orphans.MissingFiles.Select(f => f.Id).ToHashSet();
var flagChanges = new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>>();
foreach (var frame in manifest.Frames)
{
var isMissing = missing.Contains(frame.Id);
if (isMissing == frame.Flags.HasFlag(FrameFlags.Missing)) continue;
flagChanges[frame.Id] = f => f with { Flags = isMissing ? f.Flags | FrameFlags.Missing : f.Flags & ~FrameFlags.Missing };
}
if (flagChanges.Count == 0 && orphans.UnindexedFiles.Count == 0) return null;
var updated = flagChanges.Count == 0 ? manifest : SequenceFrames.Update(manifest, flagChanges);
var (added, first) = ReadFrames(sequence, orphans.UnindexedFiles, cancellationToken);
if (updated.Camera is null && first is not null)
updated = updated with { Camera = new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, Lens = updated.Lens ?? first.LensModel };
return SequenceFrames.Insert(updated, added);
}
private static (List<SidecarFrame> Frames, FrameMetadata? First) ReadFrames(SequencePaths sequence, IReadOnlyList<string> files, CancellationToken cancellationToken)
{
var frames = new List<SidecarFrame>(files.Count);
FrameMetadata? first = null;
foreach (var file in files)
{
cancellationToken.ThrowIfCancellationRequested();
var path = sequence.File(file);
try
{
var metadata = MetadataReader.Read(path, cancellationToken);
if (first is null || (metadata.CapturedAt < first.CapturedAt && metadata.Make is not null)) first = metadata;
frames.Add(SequenceFrames.FromMetadata(Guid.CreateVersion7(), file, Path.GetFileName(file), new FileInfo(path).Length, null, metadata));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Skipped unreadable photo {sequence.ArchiveRelative(file)}", ex);
}
}
return (frames, first);
}
}
@@ -0,0 +1,86 @@
using Titano.Core.Metadata;
using Titano.Core.Model;
namespace Titano.Core.Archiving;
/// <summary>Edits of the frame list of a <see cref="SequenceManifest"/>; frames stay in capture order with gap-free indexes.</summary>
public static class SequenceFrames
{
/// <summary>Capture time (unknown last), then file name.</summary>
public static readonly IComparer<SidecarFrame> CaptureOrder = Comparer<SidecarFrame>.Create((a, b) =>
{
var byTime = (a.CapturedAt ?? DateTime.MaxValue).CompareTo(b.CapturedAt ?? DateTime.MaxValue);
return byTime != 0 ? byTime : string.CompareOrdinal(a.File, b.File);
});
/// <summary>A frame record from light metadata.</summary>
/// <param name="file">Path relative to the sequence folder.</param>
/// <param name="fallbackCapture">Used when the file records no capture time of its own (only a file-system date).</param>
public static SidecarFrame FromMetadata(Guid id, string file, string? original, long size, string? sha256, FrameMetadata metadata,
DateTime? fallbackCapture = null)
{
var useFallback = fallbackCapture is not null && metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown;
return new SidecarFrame
{
Id = id,
File = file,
Original = original,
Size = size,
Sha256 = sha256?.ToLowerInvariant(),
CapturedAt = useFallback ? DateTime.SpecifyKind(fallbackCapture!.Value, DateTimeKind.Unspecified) : metadata.CapturedAt,
UtcOffset = useFallback ? null : metadata.UtcOffsetMinutes,
TimeSource = useFallback ? TimestampSource.Unknown : metadata.TimestampSource,
Exposure = metadata.ExposureTime,
FNumber = metadata.FNumber,
Iso = metadata.Iso,
Focal = metadata.FocalLength,
Bias = metadata.ExposureBias,
Wb = metadata.WhiteBalanceMode,
Orientation = metadata.Orientation,
W = metadata.Width,
H = metadata.Height,
Format = metadata.Format,
PreviewOffset = metadata.LargestPreview?.Offset,
PreviewLength = metadata.LargestPreview?.Length,
};
}
/// <summary>
/// Merges <paramref name="added"/> into the frame list by capture time (existing frames keep their relative order),
/// renumbers indexes and bumps the frames version.
/// </summary>
public static SequenceManifest Insert(SequenceManifest manifest, IReadOnlyList<SidecarFrame> added)
{
if (added.Count == 0) return manifest;
var incoming = added.Order(CaptureOrder).ToList();
var merged = new List<SidecarFrame>(manifest.Frames.Count + incoming.Count);
var next = 0;
foreach (var frame in manifest.Frames)
{
while (next < incoming.Count && CaptureOrder.Compare(incoming[next], frame) < 0) merged.Add(incoming[next++]);
merged.Add(frame);
}
while (next < incoming.Count) merged.Add(incoming[next++]);
return WithFrames(manifest, merged);
}
/// <summary>Replaces the frame list (indexes renumbered) and bumps the frames version.</summary>
public static SequenceManifest WithFrames(SequenceManifest manifest, IReadOnlyList<SidecarFrame> frames)
{
var renumbered = new SidecarFrame[frames.Count];
for (var i = 0; i < frames.Count; i++) renumbered[i] = frames[i].I == i ? frames[i] : frames[i] with { I = i };
return manifest with
{
Frames = renumbered,
Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
};
}
/// <summary>Applies per-frame changes by id (flags, hashes); frames not in <paramref name="changes"/> are kept.</summary>
public static SequenceManifest Update(SequenceManifest manifest, IReadOnlyDictionary<Guid, Func<SidecarFrame, SidecarFrame>> changes)
{
var frames = manifest.Frames.Select(f => changes.TryGetValue(f.Id, out var change) ? change(f) : f).ToList();
return manifest with { Frames = frames, UpdatedAt = DateTimeOffset.UtcNow };
}
}
@@ -0,0 +1,117 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Titano.Core.Imaging;
using Titano.Core.Model;
namespace Titano.Core.Archiving;
/// <summary>
/// <c>.titano/sequence.json</c>, the canonical record of one sequence (BRIEF §7). The database is rebuilt from it,
/// so it carries the ids of the sequence and of every frame. Fields this build does not know are kept in
/// <see cref="Extra"/> and written back unchanged.
/// </summary>
public sealed record SequenceManifest
{
public const int CurrentSchemaVersion = 1;
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
public required Guid Id { get; init; }
public string? Title { get; init; }
public SidecarCamera? Camera { get; init; }
public string? Lens { get; init; }
/// <summary>Frames in capture order; <see cref="SidecarFrame.I"/> equals the position.</summary>
public IReadOnlyList<SidecarFrame> Frames { get; init; } = [];
public IReadOnlyList<string> Tags { get; init; } = [];
public int Rating { get; init; }
public string? Notes { get; init; }
/// <summary><see cref="SequenceStatus"/> bits, stored as a number so bits of newer builds survive.</summary>
public SequenceStatus Status { get; init; }
public ManifestVersions Versions { get; init; } = new();
/// <summary>User id or client name that created the sequence.</summary>
public string? CreatedBy { get; init; }
public DateTimeOffset Created { get; init; }
public DateTimeOffset UpdatedAt { get; init; }
public DateTimeOffset? VerifiedAt { get; init; }
/// <summary>Session keys of automatic grouping (uploads, card imports) that resolve to this sequence.</summary>
public IReadOnlyList<string> SessionKeys { get; init; } = [];
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
public sealed record SidecarCamera
{
public string? Make { get; init; }
public string? Model { get; init; }
public string? Serial { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>Counters the sync protocol compares (BRIEF §14.5).</summary>
public sealed record ManifestVersions
{
public int Settings { get; init; }
public int Frames { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
/// <summary>One frame of <see cref="SequenceManifest"/>; property names follow BRIEF §7.</summary>
public sealed record SidecarFrame
{
public required Guid Id { get; init; }
/// <summary>Index in capture order, from 0.</summary>
public int I { get; init; }
/// <summary>Path relative to the sequence folder, forward slashes (<c>originals/…</c> for imported frames).</summary>
public required string File { get; init; }
/// <summary>Name the file had on the card or upload source.</summary>
public string? Original { get; init; }
public long Size { get; init; }
/// <summary>Lower-case hex SHA-256; null until a verify hashes an adopted file.</summary>
public string? Sha256 { get; init; }
/// <summary>Camera wall-clock time with sub-seconds.</summary>
public DateTime? CapturedAt { get; init; }
public int? UtcOffset { get; init; }
[JsonConverter(typeof(TolerantEnumConverter<TimestampSource>))]
public TimestampSource TimeSource { get; init; }
public double? Exposure { get; init; }
public double? FNumber { get; init; }
public int? Iso { get; init; }
public double? Focal { get; init; }
public double? Bias { get; init; }
public int? Wb { get; init; }
public int Orientation { get; init; } = 1;
public int W { get; init; }
public int H { get; init; }
[JsonConverter(typeof(TolerantEnumConverter<ImageFileFormat>))]
public ImageFileFormat Format { get; init; }
public long? PreviewOffset { get; init; }
public long? PreviewLength { get; init; }
/// <summary><see cref="FrameFlags"/> bits, stored as a number so bits of newer builds survive.</summary>
public FrameFlags Flags { get; init; }
[JsonExtensionData]
public Dictionary<string, JsonElement>? Extra { get; set; }
}
+389
View File
@@ -0,0 +1,389 @@
using System.Buffers.Binary;
using Titano.Core.Hashing;
namespace Titano.Core.Archiving;
/// <summary>One thumbnail of a pack: where its JPEG lies and which source it was made from.</summary>
/// <param name="SourceHash">32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale.</param>
public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc);
/// <summary>
/// <c>thumbs.pack</c>: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence
/// instead of thousands. Little-endian blocks after a 16-byte header:
/// <list type="bullet">
/// <item><c>TENT</c> entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes.</item>
/// <item><c>TIDX</c> index: count, one row per frame (latest entry wins), CRC-32 of the rows.</item>
/// <item><c>TFTR</c> footer (last 16 bytes): offset of the index block and its CRC.</item>
/// </list>
/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index
/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall
/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it).
/// </summary>
public static class ThumbPack
{
public const int HashLength = 32;
internal const int HeaderLength = 16;
internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4;
internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4;
internal const int FooterLength = 16;
internal const ushort Version = 1;
/// <summary>Largest thumbnail accepted; anything bigger in a header is corruption.</summary>
internal const int MaxEntryLength = 16 << 20;
internal static ReadOnlySpan<byte> Magic => "TTPK"u8;
internal static ReadOnlySpan<byte> EntryTag => "TENT"u8;
internal static ReadOnlySpan<byte> IndexTag => "TIDX"u8;
internal static ReadOnlySpan<byte> FooterTag => "TFTR"u8;
internal static void WriteHeader(Stream stream)
{
Span<byte> header = stackalloc byte[HeaderLength];
header.Clear();
Magic.CopyTo(header);
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
stream.Write(header);
}
internal static bool HasValidHeader(Stream stream)
{
if (stream.Length < HeaderLength) return false;
Span<byte> header = stackalloc byte[HeaderLength];
stream.Position = 0;
stream.ReadExactly(header);
return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version;
}
/// <summary>
/// The latest entry per frame and where the valid content ends: after the footer when the index is intact
/// (<paramref name="indexOffset"/> = start of that index), otherwise after the last complete block.
/// </summary>
internal static Dictionary<int, ThumbEntry> Load(Stream stream, out long validEnd, out long? indexOffset)
{
indexOffset = null;
if (TryLoadIndex(stream, out var entries, out var offset))
{
validEnd = stream.Length;
indexOffset = offset;
return entries;
}
return Scan(stream, out validEnd);
}
private static bool TryLoadIndex(Stream stream, out Dictionary<int, ThumbEntry> entries, out long indexOffset)
{
entries = [];
indexOffset = 0;
var length = stream.Length;
if (length < HeaderLength + 12 + FooterLength) return false;
Span<byte> footer = stackalloc byte[FooterLength];
stream.Position = length - FooterLength;
stream.ReadExactly(footer);
if (!footer[..4].SequenceEqual(FooterTag)) return false;
indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]);
if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false;
if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false;
Span<byte> head = stackalloc byte[8];
stream.Position = indexOffset;
stream.ReadExactly(head);
var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]);
if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false;
var rows = new byte[count * IndexRowLength + 4];
stream.ReadExactly(rows);
if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false;
for (var i = 0; i < count; i++)
{
var row = rows.AsSpan(i * IndexRowLength, IndexRowLength);
var entry = new ThumbEntry(
BinaryPrimitives.ReadInt32LittleEndian(row),
BinaryPrimitives.ReadInt64LittleEndian(row[4..]),
BinaryPrimitives.ReadInt32LittleEndian(row[12..]),
BinaryPrimitives.ReadUInt16LittleEndian(row[16..]),
BinaryPrimitives.ReadUInt16LittleEndian(row[18..]),
row.Slice(20, HashLength).ToArray(),
BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..]));
if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false;
entries[entry.FrameIndex] = entry;
}
return true;
}
/// <summary>Walks block headers from the start; stops at the first incomplete or unknown block.</summary>
private static Dictionary<int, ThumbEntry> Scan(Stream stream, out long validEnd)
{
var entries = new Dictionary<int, ThumbEntry>();
var length = stream.Length;
var position = (long)HeaderLength;
var header = new byte[EntryHeaderLength];
while (position + 4 <= length)
{
stream.Position = position;
stream.ReadExactly(header.AsSpan(0, 4));
var tag = header.AsSpan(0, 4);
long next;
if (tag.SequenceEqual(EntryTag))
{
if (position + EntryHeaderLength > length) break;
stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4));
var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength));
if (size is < 0 or > MaxEntryLength) break;
next = position + EntryHeaderLength + size;
if (next > length) break;
var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size,
BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)),
header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength)));
}
else if (tag.SequenceEqual(IndexTag))
{
if (position + 8 > length) break;
stream.ReadExactly(header.AsSpan(4, 4));
var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
if (count < 0) break;
next = position + 8 + (long)count * IndexRowLength + 4;
if (next > length) break;
}
else if (tag.SequenceEqual(FooterTag))
{
next = position + FooterLength;
if (next > length) break;
}
else
{
break;
}
position = next;
}
validEnd = position;
return entries;
}
/// <summary>Writes one entry block at the stream position and returns its entry.</summary>
internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
{
var crc = Crc32.Compute(jpeg);
Span<byte> header = stackalloc byte[EntryHeaderLength];
EntryTag.CopyTo(header);
BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex);
BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width);
BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height);
sourceHash.CopyTo(header[12..]);
BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length);
BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc);
var payload = stream.Position + EntryHeaderLength;
stream.Write(header);
stream.Write(jpeg);
return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc);
}
internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection<ThumbEntry> entries)
{
var indexOffset = stream.Position;
var block = new byte[8 + entries.Count * IndexRowLength + 4];
IndexTag.CopyTo(block);
BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count);
var i = 0;
foreach (var e in entries.OrderBy(e => e.FrameIndex))
{
var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength);
BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex);
BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset);
BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length);
BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width);
BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height);
e.SourceHash.CopyTo(row[20..]);
BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc);
}
var rowsLength = entries.Count * IndexRowLength;
BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength)));
stream.Write(block);
Span<byte> footer = stackalloc byte[FooterLength];
FooterTag.CopyTo(footer);
BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset);
BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12]));
stream.Write(footer);
}
/// <summary>Size of a pack holding only <paramref name="entries"/>; the rest of a file is reclaimable garbage.</summary>
public static long LiveBytes(IEnumerable<ThumbEntry> entries)
{
long count = 0, payload = 0;
foreach (var e in entries)
{
count++;
payload += EntryHeaderLength + e.Length;
}
return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength;
}
/// <summary>A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known.</summary>
public static byte[] Fingerprint(string? sha256, string file, long size) =>
sha256 is { Length: HashLength * 2 }
? Convert.FromHexString(sha256)
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
}
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
public sealed class ThumbPackReader : IDisposable
{
private readonly FileStream? _stream;
private readonly Lock _gate = new();
private ThumbPackReader(FileStream? stream, Dictionary<int, ThumbEntry> entries)
{
_stream = stream;
Entries = entries;
}
/// <summary>The latest entry of every frame that has one.</summary>
public IReadOnlyDictionary<int, ThumbEntry> Entries { get; }
/// <summary>Bytes held by replaced entries (reclaimed by <see cref="ThumbPackWriter.Compact"/>).</summary>
public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values));
public long Length => _stream?.Length ?? 0;
/// <summary>Opens a pack; a missing or damaged pack reads as empty.</summary>
public static ThumbPackReader Open(string path)
{
if (!File.Exists(path)) return new ThumbPackReader(null, []);
var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess);
try
{
var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : [];
return new ThumbPackReader(stream, entries);
}
catch
{
stream.Dispose();
throw;
}
}
/// <summary>The JPEG of a frame, or null when absent or damaged (CRC mismatch).</summary>
public byte[]? Read(int frameIndex)
{
if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null;
var data = new byte[entry.Length];
lock (_gate)
{
if (entry.Offset + entry.Length > _stream.Length) return null;
_stream.Position = entry.Offset;
_stream.ReadExactly(data);
}
return Crc32.Compute(data) == entry.Crc ? data : null;
}
public void Dispose() => _stream?.Dispose();
}
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it).</summary>
public sealed class ThumbPackWriter : IDisposable
{
private readonly FileStream _stream;
private readonly Dictionary<int, ThumbEntry> _entries;
private long? _indexOffset;
private bool _dirty;
private ThumbPackWriter(FileStream stream, Dictionary<int, ThumbEntry> entries, long? indexOffset)
{
_stream = stream;
_entries = entries;
_indexOffset = indexOffset;
}
public IReadOnlyDictionary<int, ThumbEntry> Entries => _entries;
/// <summary>Opens or creates a pack, dropping an incomplete tail left by a crash.</summary>
public static ThumbPackWriter Open(string path)
{
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16);
try
{
if (!ThumbPack.HasValidHeader(stream))
{
// Not a pack (or a header torn by a crash): thumbnails are regenerable, start over.
stream.SetLength(0);
ThumbPack.WriteHeader(stream);
return new ThumbPackWriter(stream, [], null);
}
var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset);
if (validEnd < stream.Length) stream.SetLength(validEnd);
return new ThumbPackWriter(stream, entries, indexOffset);
}
catch
{
stream.Dispose();
throw;
}
}
/// <summary>Appends a thumbnail; it replaces any earlier entry of the frame once committed.</summary>
public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
{
ArgumentOutOfRangeException.ThrowIfNegative(frameIndex);
if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash));
if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg));
if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width));
// The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk).
_stream.Position = _indexOffset ?? _stream.Length;
if (_indexOffset is { } offset) _stream.SetLength(offset);
_indexOffset = null;
_entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg);
_dirty = true;
}
/// <summary>Writes the index and footer and flushes to disk.</summary>
public void Commit()
{
if (!_dirty) return;
_stream.Position = _stream.Length;
var indexOffset = _stream.Position;
ThumbPack.WriteIndexAndFooter(_stream, _entries.Values);
_stream.Flush(flushToDisk: true);
_indexOffset = indexOffset;
_dirty = false;
}
/// <summary>Rewrites the pack with only its live entries (atomically, so readers keep a consistent file).</summary>
public static void Compact(string path)
{
var temp = IO.AtomicFile.TempPathFor(path);
try
{
using (var source = ThumbPackReader.Open(path))
using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
{
ThumbPack.WriteHeader(target);
var entries = new List<ThumbEntry>(source.Entries.Count);
foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex))
{
if (source.Read(entry.FrameIndex) is { } jpeg)
entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg));
}
ThumbPack.WriteIndexAndFooter(target, entries);
}
IO.AtomicFile.Commit(temp, path);
}
finally
{
if (File.Exists(temp)) File.Delete(temp);
}
}
public void Dispose()
{
try
{
Commit();
}
finally
{
_stream.Dispose();
}
}
}
+90
View File
@@ -0,0 +1,90 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
namespace Titano.Core.Import;
/// <summary>A card file that was imported, with the hash recorded at import and where its copy lives.</summary>
/// <param name="SourcePath">Path below the card root, forward slashes.</param>
/// <param name="CopyPath">Absolute path of the archive (or folder) copy.</param>
public sealed record EraseCandidate(string SourcePath, long Size, string Sha256, string CopyPath);
public enum EraseVerdict
{
/// <summary>Card file and copy both hash to the recorded SHA-256: safe to delete from the card.</summary>
Delete,
SourceMissing,
/// <summary>The card file no longer matches what was imported.</summary>
SourceChanged,
CopyMissing,
/// <summary>The copy no longer matches: the card file is the only good one left.</summary>
CopyMismatch,
}
/// <param name="SourceModifiedUtc">Card file time when checked; execution refuses a file touched since.</param>
public sealed record EraseItem(EraseCandidate Candidate, EraseVerdict Verdict, DateTime? SourceModifiedUtc);
/// <summary>The dry run of "delete imported" (BRIEF §12.4, §13): exactly the files that would be deleted, and why
/// the others are kept.</summary>
public sealed record ErasePlan(string SourceRoot, IReadOnlyList<EraseItem> Items)
{
public IEnumerable<EraseItem> ToDelete => Items.Where(i => i.Verdict == EraseVerdict.Delete);
}
public sealed record EraseResult(int Deleted, IReadOnlyList<EraseItem> Kept);
/// <summary>
/// Deletes imported photos from a card only after re-hashing both the card file and its copy against the hash
/// recorded at import (BRIEF §0.2 rule 8). Never formats (that is the OS dialog) and never touches folders.
/// </summary>
public static class CardEraser
{
public static async Task<ErasePlan> PlanAsync(string sourceRoot, IReadOnlyList<EraseCandidate> candidates, IProgress<int>? progress = null,
CancellationToken cancellationToken = default)
{
var items = new List<EraseItem>(candidates.Count);
for (var i = 0; i < candidates.Count; i++)
{
var c = candidates[i];
var source = new FileInfo(Path.Combine(sourceRoot, c.SourcePath));
var verdict = !source.Exists ? EraseVerdict.SourceMissing
: !File.Exists(c.CopyPath) ? EraseVerdict.CopyMissing
: !await MatchesAsync(c.CopyPath, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.CopyMismatch
: !await MatchesAsync(source.FullName, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.SourceChanged
: EraseVerdict.Delete;
items.Add(new EraseItem(c, verdict, source.Exists ? source.LastWriteTimeUtc : null));
progress?.Report(i + 1);
}
return new ErasePlan(Path.GetFullPath(sourceRoot), items);
}
/// <summary>Deletes the plan's <see cref="EraseVerdict.Delete"/> files whose size and time are unchanged since planning.</summary>
public static EraseResult Execute(ErasePlan plan, CancellationToken cancellationToken = default)
{
var deleted = 0;
var kept = plan.Items.Where(i => i.Verdict != EraseVerdict.Delete).ToList();
foreach (var item in plan.ToDelete)
{
cancellationToken.ThrowIfCancellationRequested();
var source = new FileInfo(Path.Combine(plan.SourceRoot, item.Candidate.SourcePath));
var copy = new FileInfo(item.Candidate.CopyPath);
if (!source.Exists || source.Length != item.Candidate.Size || source.LastWriteTimeUtc != item.SourceModifiedUtc
|| !copy.Exists || copy.Length != item.Candidate.Size)
{
kept.Add(item with { Verdict = EraseVerdict.SourceChanged });
continue;
}
source.Delete();
deleted++;
Log.Info("import", $"Deleted verified {item.Candidate.SourcePath} from the card");
}
return new EraseResult(deleted, kept);
}
private static async Task<bool> MatchesAsync(string path, EraseCandidate candidate, CancellationToken cancellationToken) =>
new FileInfo(path).Length == candidate.Size
&& string.Equals(await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false), candidate.Sha256, StringComparison.OrdinalIgnoreCase);
}
+87
View File
@@ -0,0 +1,87 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Metadata;
namespace Titano.Core.Import;
/// <summary>A photo file on a card or source folder.</summary>
/// <param name="RelativePath">Path below the source root, forward slashes (the card import history key).</param>
public sealed record CardFile(string RelativePath, long Size, DateTime ModifiedUtc);
/// <summary>A card photo with its light metadata.</summary>
public sealed record CardPhoto(CardFile File, FrameMetadata Metadata)
{
/// <summary>Capture time, or the file time when the photo records none.</summary>
public DateTime CaptureTime => Metadata.CapturedAt ?? File.ModifiedUtc;
}
/// <summary>A file that could not be read while planning; it is listed, never a reason to stop.</summary>
public sealed record ImportProblem(string RelativePath, string Message);
/// <summary>Finds photos on a card (BRIEF §12.4, §13): the <c>DCIM</c> tree when present, otherwise the root.</summary>
public static class CardScanner
{
private const int MaxDepth = 12;
private static readonly EnumerationOptions Listing = new()
{
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System | FileAttributes.Hidden,
RecurseSubdirectories = false,
MatchCasing = MatchCasing.CaseInsensitive,
};
/// <summary>
/// Photos below <paramref name="root"/>: everything under <c>DCIM</c> (camera folders) when the card has one,
/// otherwise the root's photos and, with <paramref name="includeSubfolders"/>, its subfolders'. Ordered by path.
/// </summary>
public static IReadOnlyList<CardFile> Find(string root, bool includeSubfolders, CancellationToken cancellationToken = default)
{
var dcim = Directory.EnumerateDirectories(root, "DCIM", Listing).FirstOrDefault();
var start = dcim ?? root;
var recurse = dcim is not null || includeSubfolders;
var files = new List<CardFile>();
var pending = new Stack<(string Path, int Depth)>();
pending.Push((start, 0));
while (pending.Count > 0)
{
cancellationToken.ThrowIfCancellationRequested();
var (directory, depth) = pending.Pop();
foreach (var file in new DirectoryInfo(directory).EnumerateFiles("*", Listing))
{
if (ArchiveScanner.IsPhoto(file.Name))
files.Add(new CardFile(Path.GetRelativePath(root, file.FullName).Replace('\\', '/'), file.Length, file.LastWriteTimeUtc));
}
if (!recurse || depth >= MaxDepth) continue;
foreach (var child in Directory.EnumerateDirectories(directory, "*", Listing))
{
if (!Path.GetFileName(child).StartsWith('.')) pending.Push((child, depth + 1));
}
}
files.Sort((a, b) => string.CompareOrdinal(a.RelativePath, b.RelativePath));
return files;
}
/// <summary>Reads light metadata of every file; unreadable files become <paramref name="problems"/>.</summary>
public static IReadOnlyList<CardPhoto> ReadMetadata(string root, IReadOnlyList<CardFile> files, List<ImportProblem> problems,
IProgress<int>? progress = null, CancellationToken cancellationToken = default)
{
var photos = new List<CardPhoto>(files.Count);
for (var i = 0; i < files.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var file = files[i];
try
{
photos.Add(new CardPhoto(file, MetadataReader.Read(Path.Combine(root, file.RelativePath), cancellationToken)));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("import", $"Cannot read {file.RelativePath}", ex);
problems.Add(new ImportProblem(file.RelativePath, ex.Message));
}
progress?.Report(i + 1);
}
return photos;
}
}
@@ -0,0 +1,42 @@
using Titano.Core.Archiving;
namespace Titano.Core.Import;
/// <summary>
/// Import destination "Folder" (BRIEF §12.4): photos go to <c>root/&lt;folder&gt;/&lt;name&gt;</c> without sidecars or index.
/// A name already taken by identical content is a duplicate; by other content, the photo gets a suffix. Nothing at the
/// destination is ever replaced.
/// </summary>
public sealed class FolderImportSink(string root) : IImportSink
{
public string StagingDirectory { get; } = Path.Combine(Path.GetFullPath(root), ArchiveLayout.MetaFolder, ArchiveLayout.StagingFolder);
public async Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken)
{
var folder = Path.Combine(Path.GetFullPath(root), session.Folder.Replace('/', Path.DirectorySeparatorChar));
Directory.CreateDirectory(folder);
var results = new List<SinkCommit>(files.Count);
foreach (var file in files)
{
var name = file.Item.FileName;
var stem = Path.GetFileNameWithoutExtension(name);
var duplicate = false;
for (var n = 2; File.Exists(Path.Combine(folder, name)); n++)
{
if (await IsSameAsync(Path.Combine(folder, name), file, cancellationToken).ConfigureAwait(false))
{
duplicate = true;
break;
}
name = $"{stem}_{n}{Path.GetExtension(file.Item.FileName)}";
}
if (!duplicate) File.Move(file.StagedPath, Path.Combine(folder, name), overwrite: false);
results.Add(new SinkCommit(null, session.Folder + "/" + name, duplicate));
}
return results;
}
private static async Task<bool> IsSameAsync(string existing, StagedFile file, CancellationToken cancellationToken) =>
new FileInfo(existing).Length == file.Item.Size
&& await ContentHash.Sha256Async(existing, null, cancellationToken).ConfigureAwait(false) == file.Sha256;
}
+148
View File
@@ -0,0 +1,148 @@
using Titano.Core.Diagnostics;
using Titano.Core.Localization;
namespace Titano.Core.Import;
/// <summary>The card or source folder disappeared mid-import (card removed, share lost). The import can resume.</summary>
public sealed class ImportSourceUnavailableException(string root, Exception? inner = null)
: IOException(Loc.L("Archive.Error.SourceGone", root), inner)
{
public string Root { get; } = root;
}
/// <summary>A verified copy waiting in the sink's staging folder.</summary>
public sealed record StagedFile(PlannedItem Item, string StagedPath, string Sha256);
/// <summary>Where a staged file ended up.</summary>
/// <param name="FrameId">The archive frame (null for a plain folder destination).</param>
/// <param name="Duplicate">The destination already held this content; the staged copy was not used.</param>
public sealed record SinkCommit(Guid? FrameId, string RelativePath, bool Duplicate);
/// <summary>Destination of an import: the archive, a plain folder, or a server (BRIEF §12.4).</summary>
public interface IImportSink
{
/// <summary>Folder for verified copies, on the destination volume so commits are renames.</summary>
string StagingDirectory { get; }
/// <summary>Moves staged files of one session into place; returns one result per file, in order. Files it does
/// not move stay where they are and the executor deletes them.</summary>
Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken);
}
public enum ImportItemStatus
{
Copied,
/// <summary>Not copied: imported before, or already in the archive.</summary>
Skipped,
/// <summary>Copied and verified, but the destination already held the same content.</summary>
Duplicate,
Failed,
}
public sealed record ImportItemResult(PlannedSession Session, PlannedItem Item, ImportItemStatus Status, string? Sha256, SinkCommit? Commit, string? Error);
/// <summary>Live figures for the job bar: items and bytes (throughput and ETA come from bytes).</summary>
public sealed record ImportProgress(int ItemsDone, int ItemsTotal, long BytesDone, long BytesTotal, int Copied, int Skipped, int Failed, string? Current);
/// <summary>Results committed since the previous batch; <see cref="NextItem"/> is the resume point once they are recorded.</summary>
public sealed record ImportBatch(IReadOnlyList<ImportItemResult> Results, int NextItem, ImportProgress Progress);
/// <summary>
/// Executes an <see cref="ImportPlan"/> (BRIEF §12.4, §13): copies each photo to staging with hashing and read-back
/// verification, commits verified copies to the sink in batches, reports per-file errors without stopping, and can
/// resume from any batch boundary. When the source disappears it commits what is already verified and throws
/// <see cref="ImportSourceUnavailableException"/>.
/// </summary>
public sealed class ImportExecutor(ImportPlan plan, IImportSink sink)
{
private const int BatchSize = 32;
/// <param name="startAt">Number of plan items already done (the last <see cref="ImportBatch.NextItem"/> recorded).</param>
/// <param name="committed">Called after each batch; record the results, then checkpoint <see cref="ImportBatch.NextItem"/>.</param>
/// <param name="progress">Called before and after each copy (throughput and ETA come from the bytes).</param>
public async Task<ImportProgress> RunAsync(int startAt, bool verify, Func<ImportBatch, ValueTask> committed,
Func<ImportProgress, ValueTask>? progress = null, CancellationToken cancellationToken = default)
{
var items = plan.Sessions.SelectMany(s => s.Items.Select(i => (Session: s, Item: i))).ToList();
var totalBytes = items.Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size);
var bytesDone = items.Take(startAt).Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size);
int copied = 0, skipped = 0, failed = 0;
var results = new List<ImportItemResult>();
var staged = new List<StagedFile>();
PlannedSession? stagedSession = null;
var next = startAt;
ImportProgress Snapshot(string? current) => new(next, items.Count, bytesDone, totalBytes, copied, skipped, failed, current);
async Task FlushAsync()
{
if (staged.Count > 0)
{
var commits = await sink.CommitAsync(stagedSession!, staged, cancellationToken).ConfigureAwait(false);
for (var i = 0; i < staged.Count; i++)
{
var duplicate = commits[i].Duplicate;
results.Add(new ImportItemResult(stagedSession!, staged[i].Item, duplicate ? ImportItemStatus.Duplicate : ImportItemStatus.Copied,
staged[i].Sha256, commits[i], null));
if (duplicate) skipped++;
else copied++;
DeleteLeftover(staged[i].StagedPath);
}
staged.Clear();
stagedSession = null;
}
if (results.Count == 0) return;
await committed(new ImportBatch([.. results], next, Snapshot(null))).ConfigureAwait(false);
results.Clear();
}
Directory.CreateDirectory(sink.StagingDirectory);
for (; next < items.Count; next++)
{
cancellationToken.ThrowIfCancellationRequested();
var (session, item) = items[next];
if ((stagedSession is not null && !ReferenceEquals(stagedSession, session)) || staged.Count + results.Count >= BatchSize)
await FlushAsync().ConfigureAwait(false);
if (item.Action != ImportAction.Copy)
{
skipped++;
results.Add(new ImportItemResult(session, item, ImportItemStatus.Skipped, item.Sha256, null, null));
continue;
}
var source = Path.Combine(plan.SourceRoot, item.SourcePath);
if (progress is not null) await progress(Snapshot(item.SourcePath)).ConfigureAwait(false);
var target = Path.Combine(sink.StagingDirectory, $"{plan.Id:N}-{next:D6}{Path.GetExtension(item.FileName)}");
DeleteLeftover(target);
try
{
if (new FileInfo(source) is not { Exists: true } info || info.Length != item.Size)
throw new IOException(Loc.L("Archive.Error.SourceChanged", item.SourcePath));
var sha = await VerifiedCopy.CopyAsync(source, target, verify, read => bytesDone += read, cancellationToken).ConfigureAwait(false);
staged.Add(new StagedFile(item, target, sha));
stagedSession = session;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
if (!Directory.Exists(plan.SourceRoot))
{
await FlushAsync().ConfigureAwait(false);
throw new ImportSourceUnavailableException(plan.SourceRoot, ex);
}
Log.Warning("import", $"Copy of {item.SourcePath} failed", ex);
failed++;
results.Add(new ImportItemResult(session, item, ImportItemStatus.Failed, null, null, ex.Message));
}
}
await FlushAsync().ConfigureAwait(false);
return Snapshot(null);
}
private static void DeleteLeftover(string staged)
{
// Staged files are this import's own copies of card files; the originals stay on the card.
if (File.Exists(staged)) File.Delete(staged);
}
}
+118
View File
@@ -0,0 +1,118 @@
using System.Text.Json.Serialization;
using Titano.Core.Imaging;
using Titano.Core.Naming;
namespace Titano.Core.Import;
/// <summary>What to do with a photo that is already in the archive (same SHA-256).</summary>
public enum DuplicatePolicy
{
/// <summary>Leave it on the card; the archive copy is identical.</summary>
Skip,
/// <summary>Import it again into this session's sequence (its name gets a suffix if taken).</summary>
ImportAgain,
}
/// <summary>Card import settings (BRIEF §12.4, §13); the defaults are the brief's.</summary>
public sealed record ImportOptions
{
public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
public string FileTemplate { get; init; } = TemplateDefaults.File;
public TimeSpan SessionGap { get; init; } = TimeSpan.FromMinutes(45);
public int MinimumSessionFrames { get; init; } = 8;
public bool IncludeSubfolders { get; init; } = true;
public bool VerifyCopies { get; init; } = true;
public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
/// <summary>Card paths where the user split a session (a new session starts at that photo).</summary>
public IReadOnlySet<string> SplitBefore { get; init; } = new HashSet<string>();
/// <summary>Card paths where the user merged a session into the previous one.</summary>
public IReadOnlySet<string> MergeAt { get; init; } = new HashSet<string>();
/// <summary>Titles given to sessions, by session key; used as the sequence title and the <c>{title}</c> token.</summary>
public IReadOnlyDictionary<string, string> Titles { get; init; } = new Dictionary<string, string>();
}
public enum ImportAction
{
Copy,
/// <summary>Imported from this card before (same path, size and time).</summary>
SkipImported,
/// <summary>Already in the archive (same size and capture time, then the same SHA-256).</summary>
SkipInArchive,
}
/// <summary>One photo of the plan and its destination name.</summary>
public sealed record PlannedItem
{
/// <summary>Path below the source root, forward slashes.</summary>
public required string SourcePath { get; init; }
public long Size { get; init; }
public DateTime ModifiedUtc { get; init; }
public DateTime CapturedAt { get; init; }
public required string FileName { get; init; }
public ImportAction Action { get; init; }
/// <summary>The archive frame this photo duplicates (<see cref="ImportAction.SkipInArchive"/>, or imported again).</summary>
public Guid? ExistingFrameId { get; init; }
/// <summary>SHA-256 computed while planning (duplicate check), if any.</summary>
public string? Sha256 { get; init; }
}
/// <summary>A session of the plan: one camera body and file format, one sequence in the archive.</summary>
public sealed record PlannedSession
{
/// <summary>1-based, in order of start time across the import.</summary>
public int Number { get; init; }
/// <summary>Stable key of the session; re-planning the same photos yields the same key, so an interrupted import
/// resumes into the same sequence.</summary>
public required string SessionKey { get; init; }
public string? CameraMake { get; init; }
public string? CameraModel { get; init; }
public string? CameraSerial { get; init; }
public ImageFileFormat Format { get; init; }
public DateTime Start { get; init; }
public DateTime End { get; init; }
/// <summary>Destination folder relative to the destination root, forward slashes.</summary>
public required string Folder { get; init; }
public string? Title { get; init; }
public IReadOnlyList<PlannedItem> Items { get; init; } = [];
}
/// <summary>
/// The exact result of an import before any byte is written (BRIEF §12.4 "Plan before touching disk"): sessions,
/// folders, file names, what is skipped and why, and the space it needs. Executing it copies exactly this.
/// </summary>
public sealed record ImportPlan
{
public required Guid Id { get; init; }
public required string SourceRoot { get; init; }
public required string DestinationRoot { get; init; }
public DateTimeOffset Created { get; init; }
public IReadOnlyList<PlannedSession> Sessions { get; init; } = [];
public IReadOnlyList<ImportProblem> Problems { get; init; } = [];
/// <summary>Free space at the destination when planned; null when the volume does not report it.</summary>
public long? AvailableBytes { get; init; }
[JsonIgnore]
public IEnumerable<PlannedItem> Items => Sessions.SelectMany(s => s.Items);
[JsonIgnore]
public int FileCount => Sessions.Sum(s => s.Items.Count);
[JsonIgnore]
public int CopyCount => Items.Count(i => i.Action == ImportAction.Copy);
[JsonIgnore]
public long CopyBytes => Items.Where(i => i.Action == ImportAction.Copy).Sum(i => i.Size);
[JsonIgnore]
public bool HasEnoughSpace => AvailableBytes is not { } free || free >= CopyBytes;
}
+181
View File
@@ -0,0 +1,181 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Titano.Core.Archiving;
using Titano.Core.Naming;
namespace Titano.Core.Import;
/// <summary>An archive frame that may be the same photo.</summary>
public sealed record ArchiveMatch(Guid FrameId, string? Sha256);
/// <summary>What the planner needs to know about the destination (implemented over the archive index).</summary>
public interface IImportLookup
{
/// <summary>The file was imported from this card before (same path, size and time).</summary>
Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken);
/// <summary>Archive frames with this size and capture time; the planner compares hashes only for these.</summary>
Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken);
/// <summary>Folder of the sequence an earlier import of this session went to, so a re-import continues it.</summary>
Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken);
/// <summary>The folder is taken by something else (another sequence), so the session needs another name.</summary>
bool IsFolderTaken(string folder);
}
/// <summary>Turns card photos into an <see cref="ImportPlan"/> (BRIEF §12.4, §13). Nothing is written.</summary>
public static class ImportPlanner
{
public static async Task<ImportPlan> PlanAsync(string sourceRoot, string destinationRoot, IReadOnlyList<CardPhoto> photos,
IReadOnlyList<ImportProblem> problems, ImportOptions options, IImportLookup lookup, CancellationToken cancellationToken = default)
{
var folderTemplate = PathTemplate.Parse(options.FolderTemplate);
var fileTemplate = PathTemplate.Parse(options.FileTemplate);
var drafts = new List<(string Key, List<CardPhoto> Photos)>();
foreach (var body in photos.GroupBy(BodyKey, StringComparer.Ordinal))
{
var ordered = body.OrderBy(p => p.CaptureTime).ThenBy(p => p.File.RelativePath, StringComparer.Ordinal).ToList();
var ranges = SessionSplitter.Split(ordered.ConvertAll(p => p.CaptureTime), options.SessionGap, options.MinimumSessionFrames,
i => options.SplitBefore.Contains(ordered[i].File.RelativePath) ? true : options.MergeAt.Contains(ordered[i].File.RelativePath) ? false : null);
foreach (var range in ranges)
{
var members = ordered.GetRange(range.Start, range.Count);
drafts.Add((SessionKey(body.Key, members[0].CaptureTime), members));
}
}
drafts.Sort((a, b) => a.Photos[0].CaptureTime.CompareTo(b.Photos[0].CaptureTime) is var c and not 0 ? c : string.CompareOrdinal(a.Key, b.Key));
var claimedFolders = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var sessions = new List<PlannedSession>(drafts.Count);
for (var s = 0; s < drafts.Count; s++)
{
var (key, members) = drafts[s];
var first = members[0];
options.Titles.TryGetValue(key, out var title);
var context = Context(first, first.CaptureTime, s + 1, 1, title);
var folder = await lookup.FindSessionFolderAsync(key, cancellationToken).ConfigureAwait(false)
?? UniqueFolder(folderTemplate.ExpandFolder(context), claimedFolders, lookup);
claimedFolders.Add(folder);
var names = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var items = new List<PlannedItem>(members.Count);
for (var i = 0; i < members.Count; i++)
{
var photo = members[i];
var name = UniqueName(fileTemplate.ExpandFile(Context(photo, first.CaptureTime, s + 1, i + 1, title)), names);
items.Add(await PlanItemAsync(sourceRoot, photo, name, options, lookup, cancellationToken).ConfigureAwait(false));
}
sessions.Add(new PlannedSession
{
Number = s + 1,
SessionKey = key,
CameraMake = first.Metadata.Make,
CameraModel = first.Metadata.Model,
CameraSerial = first.Metadata.BodySerial,
Format = first.Metadata.Format,
Start = first.CaptureTime,
End = members[^1].CaptureTime,
Folder = folder,
Title = title,
Items = items,
});
}
return new ImportPlan
{
Id = Guid.CreateVersion7(),
SourceRoot = Path.GetFullPath(sourceRoot),
DestinationRoot = Path.GetFullPath(destinationRoot),
Created = DateTimeOffset.UtcNow,
Sessions = sessions,
Problems = problems,
AvailableBytes = FreeSpace(destinationRoot),
};
}
private static async Task<PlannedItem> PlanItemAsync(string sourceRoot, CardPhoto photo, string name, ImportOptions options, IImportLookup lookup,
CancellationToken cancellationToken)
{
var item = new PlannedItem
{
SourcePath = photo.File.RelativePath,
Size = photo.File.Size,
ModifiedUtc = photo.File.ModifiedUtc,
CapturedAt = photo.CaptureTime,
FileName = name,
};
if (await lookup.WasImportedAsync(photo.File, cancellationToken).ConfigureAwait(false))
return item with { Action = ImportAction.SkipImported };
var candidates = await lookup.FindCandidatesAsync(photo.File.Size, photo.CaptureTime, cancellationToken).ConfigureAwait(false);
if (candidates.Count == 0) return item;
var sha = await ContentHash.Sha256Async(Path.Combine(sourceRoot, photo.File.RelativePath), null, cancellationToken).ConfigureAwait(false);
var match = candidates.FirstOrDefault(c => sha.Equals(c.Sha256, StringComparison.OrdinalIgnoreCase));
if (match is null) return item with { Sha256 = sha };
return item with
{
Sha256 = sha,
ExistingFrameId = match.FrameId,
Action = options.Duplicates == DuplicatePolicy.Skip ? ImportAction.SkipInArchive : ImportAction.Copy,
};
}
private static NamingContext Context(CardPhoto photo, DateTime sessionStart, int session, int number, string? title) => new()
{
CapturedAt = photo.CaptureTime,
SessionStart = sessionStart,
Session = session,
Number = number,
NumberInSession = number,
CameraMake = photo.Metadata.Make,
CameraModel = photo.Metadata.Model,
Lens = photo.Metadata.LensModel,
Iso = photo.Metadata.Iso,
ExposureTime = photo.Metadata.ExposureTime,
FNumber = photo.Metadata.FNumber,
OriginalName = Path.GetFileNameWithoutExtension(photo.File.RelativePath),
Extension = Path.GetExtension(photo.File.RelativePath).TrimStart('.'),
Title = title,
};
/// <summary>Camera body and file format: RAW+JPEG shooting yields two sequences instead of doubled frames.</summary>
private static string BodyKey(CardPhoto p) => $"{p.Metadata.Make}|{p.Metadata.Model}|{p.Metadata.BodySerial}|{p.Metadata.Format}";
private static string SessionKey(string bodyKey, DateTime start) =>
"card-" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(bodyKey)))[..8] + "-"
+ start.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
private static string UniqueFolder(string folder, HashSet<string> claimed, IImportLookup lookup)
{
if (folder.Length == 0) folder = "Import";
var candidate = folder;
for (var n = 2; claimed.Contains(candidate) || lookup.IsFolderTaken(candidate); n++)
candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})");
return candidate;
}
/// <summary>Adds _2, _3… before the extension when the template gives two photos the same name.</summary>
internal static string UniqueName(string name, HashSet<string> taken)
{
var candidate = name;
var stem = Path.GetFileNameWithoutExtension(name);
var extension = Path.GetExtension(name);
for (var n = 2; !taken.Add(candidate); n++) candidate = string.Create(CultureInfo.InvariantCulture, $"{stem}_{n}{extension}");
return candidate;
}
private static long? FreeSpace(string path)
{
try
{
var full = Path.GetFullPath(path);
while (!Directory.Exists(full) && Path.GetDirectoryName(full) is { } parent) full = parent;
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace;
}
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
{
return null;
}
}
}
+61
View File
@@ -0,0 +1,61 @@
namespace Titano.Core.Import;
/// <summary>A run of consecutive frames forming one session: <c>[Start, Start + Count)</c>.</summary>
public readonly record struct SessionRange(int Start, int Count)
{
public int End => Start + Count;
}
/// <summary>
/// Splits a time-ordered run of captures into shooting sessions (BRIEF §13): a pause longer than the gap starts a
/// new session; a session with fewer than the minimum frames (test shots) merges into the previous one, or into the
/// next when it comes first. Breaks the user forced are never merged away; breaks the user removed never split.
/// </summary>
public static class SessionSplitter
{
/// <param name="times">Capture times in ascending order.</param>
/// <param name="userBreak">For a frame index: true = the user starts a session there, false = the user joined it
/// to the previous frame, null = automatic.</param>
public static IReadOnlyList<SessionRange> Split(IReadOnlyList<DateTime> times, TimeSpan gap, int minimumFrames, Func<int, bool?>? userBreak = null)
{
if (times.Count == 0) return [];
var groups = new List<(int Start, int Count, bool Forced)>();
var start = 0;
var forced = false;
for (var i = 1; i < times.Count; i++)
{
var decision = userBreak?.Invoke(i);
if (decision == false || (decision is null && times[i] - times[i - 1] <= gap)) continue;
groups.Add((start, i - start, forced));
start = i;
forced = decision == true;
}
groups.Add((start, times.Count - start, forced));
for (var i = 0; i < groups.Count && groups.Count > 1;)
{
var g = groups[i];
if (g.Count >= minimumFrames)
{
i++;
continue;
}
if (i > 0 && !g.Forced)
{
var previous = groups[i - 1];
groups[i - 1] = (previous.Start, previous.Count + g.Count, previous.Forced);
groups.RemoveAt(i);
}
else if (i == 0 && !groups[1].Forced)
{
groups[1] = (g.Start, g.Count + groups[1].Count, g.Forced);
groups.RemoveAt(0);
}
else
{
i++;
}
}
return groups.ConvertAll(g => new SessionRange(g.Start, g.Count));
}
}
+65
View File
@@ -0,0 +1,65 @@
using System.Security.Cryptography;
using Titano.Core.Archiving;
using Titano.Core.Localization;
namespace Titano.Core.Import;
/// <summary>The copy read back from the destination does not match what was read from the source.</summary>
public sealed class CopyVerificationException(string path)
: IOException(Loc.L("Archive.Error.CopyMismatch", Path.GetFileName(path)))
{
public string FilePath { get; } = path;
}
/// <summary>
/// Copies a file while hashing it, flushes the copy to disk, then re-reads it and compares hashes (BRIEF §12.4).
/// A failed or interrupted copy never leaves a partial file behind.
/// </summary>
public static class VerifiedCopy
{
/// <summary>Copies <paramref name="source"/> to the new file <paramref name="destination"/>; returns the SHA-256.</summary>
/// <param name="verify">Re-read the copy and compare (the default import setting).</param>
/// <param name="bytesCopied">Receives the bytes of each chunk read from the source.</param>
/// <exception cref="CopyVerificationException">The re-read copy differs.</exception>
public static async Task<string> CopyAsync(string source, string destination, bool verify, Action<long>? bytesCopied = null,
CancellationToken cancellationToken = default)
{
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(destination))!);
var created = false;
try
{
string sha;
DateTime modified;
await using (var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, ContentHash.BufferSize,
FileOptions.Asynchronous | FileOptions.SequentialScan))
await using (var output = new FileStream(destination, FileMode.CreateNew, FileAccess.Write, FileShare.None, ContentHash.BufferSize,
FileOptions.Asynchronous))
{
created = true;
modified = File.GetLastWriteTimeUtc(source);
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
var buffer = new byte[ContentHash.BufferSize];
int read;
while ((read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
{
hash.AppendData(buffer, 0, read);
await output.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
bytesCopied?.Invoke(read);
}
await output.FlushAsync(cancellationToken).ConfigureAwait(false);
output.Flush(flushToDisk: true);
sha = Convert.ToHexStringLower(hash.GetHashAndReset());
}
File.SetLastWriteTimeUtc(destination, modified);
if (verify && await ContentHash.Sha256Async(destination, null, cancellationToken).ConfigureAwait(false) != sha)
throw new CopyVerificationException(destination);
return sha;
}
catch
{
// Only our own partial or unverified copy; an existing file at the destination is never touched.
if (created && File.Exists(destination)) File.Delete(destination);
throw;
}
}
}
+174
View File
@@ -0,0 +1,174 @@
using System.Diagnostics;
using Titano.Core.Archiving;
using Titano.Core.IO;
using Titano.Core.Metadata;
using Titano.Core.Model;
namespace Titano.Data.Archiving;
/// <summary>Frames processed by one thumbnail or proxy pass over a sequence.</summary>
public sealed record ImagePassResult(int Built, int Reused, int Undecodable);
/// <summary>
/// Thumbnails (<c>thumbs.pack</c>) and proxies (<c>proxies/0001.jpg</c>) of sequences (BRIEF §8.5). Both are
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
/// frames were renumbered the stale ones are rebuilt and the rest reused.
/// </summary>
public sealed class ArchiveImages
{
private const int CommitFrames = 200;
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly FrameImageRenderer _renderer;
internal ArchiveImages(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
_renderer = new FrameImageRenderer(writer.Options.Decoder);
}
/// <param name="startFrame">Resume point inside the sequence (frames before it were done by an earlier run).</param>
/// <param name="checkpoint">Called with the next frame index whenever the pack has been committed.</param>
public async Task<ImagePassResult> BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
int built = 0, reused = 0, undecodable = 0;
// Thumbnail writers exclude each other only; ingest keeps committing meanwhile.
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath))
{
var cadence = new Cadence();
for (var i = startFrame; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = manifest.Frames[i];
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint))
{
reused++;
continue;
}
if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image)
{
pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg);
built++;
}
else
{
undecodable++;
}
if (cadence.Due())
{
pack.Commit();
await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
}
}
pack.Commit();
}
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
/// <inheritdoc cref="BuildThumbnailsAsync"/>
public async Task<ImagePassResult> BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
var folder = paths.Sequence(sequence.Folder);
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
Directory.CreateDirectory(folder.Proxies);
int built = 0, reused = 0, undecodable = 0;
var cadence = new Cadence();
for (var i = startFrame; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var frame = manifest.Frames[i];
var path = folder.Proxy(frame.I);
if (!force && IsCurrent(path, frame))
{
reused++;
continue;
}
if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image)
{
AtomicFile.WriteAllBytes(path, image.Jpeg);
built++;
}
else
{
undecodable++;
}
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
}
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
return new ImagePassResult(built, reused, undecodable);
}
/// <summary>A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale).</summary>
private static bool IsCurrent(string proxy, SidecarFrame frame)
{
if (!File.Exists(proxy)) return false;
try
{
return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return false;
}
}
/// <summary>A frame no decoder can read is skipped and counted; it never stops the pass.</summary>
private static EncodedImage? Render(Func<EncodedImage?> render)
{
try
{
return render();
}
catch (Exception ex) when (ex is IOException or InvalidDataException or NotSupportedException)
{
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
return null;
}
}
/// <summary>Commit/checkpoint rhythm: every <see cref="CommitFrames"/> rendered frames or <see cref="CommitInterval"/>.</summary>
private sealed class Cadence
{
private readonly Stopwatch _clock = Stopwatch.StartNew();
private int _frames;
public bool Due()
{
if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
_frames = 0;
_clock.Restart();
return true;
}
}
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
{
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
{
var status = complete ? m.Status | flag : m.Status & ~flag;
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
}, cancellationToken).ConfigureAwait(false);
await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
.ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
+268
View File
@@ -0,0 +1,268 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
public enum ScanMode
{
/// <summary>Only folders whose sidecar or photo folder changed size or time since the last scan.</summary>
Incremental,
/// <summary>Every folder: sidecars re-read and folders reconciled with their photos (manual rescan).</summary>
Full,
/// <summary>The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5).</summary>
Rebuild,
}
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed);
/// <summary>
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one
/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived.
/// </summary>
public sealed class ArchiveIndexer
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
internal ArchiveIndexer(TitanoDatabase db, SequenceWriter writer)
{
_db = db;
_writer = writer;
}
/// <param name="resumeAfter">Last folder completed by an interrupted run (job checkpoint), or null.</param>
/// <param name="progress">Called after each folder; persist <see cref="ScanProgress.Folder"/> as the resume point.</param>
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the index is left as it is.</exception>
public async Task<ScanSummary> ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func<ScanProgress, ValueTask> progress,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
int indexed = 0, adopted = 0;
var failed = new List<(string, string)>();
for (var i = 0; i < folders.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var folder = folders[i];
if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue;
states.TryGetValue(folder.RelativeFolder, out var state);
var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos;
if (changed)
{
try
{
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
if (snapshotWriter is not null)
{
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
}
indexed++;
if (folder.Sidecar is null) adopted++;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex);
failed.Add((folder.RelativeFolder, ex.Message));
}
}
await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false);
}
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
if (failed.Count == 0) snapshotWriter?.Commit();
return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
}
/// <summary>Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps.</summary>
internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
{
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
foreach (var key in manifest.SessionKeys)
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
/// incremental scan does not re-read what is already indexed.
/// </summary>
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var folder = paths.Sequence(sequence.Folder);
var manifest = await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceAdopter.Reconcile(folder, m, cancellationToken),
cancellationToken).ConfigureAwait(false);
await SyncAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
return manifest;
}
/// <summary>Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
private async Task<byte[]?> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
Locations locations, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(folder.RelativeFolder);
SequenceManifest manifest;
byte[]? bytes = null;
if (folder.Sidecar is not { } stamp)
{
manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
}
else
{
bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await File.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false);
try
{
manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath);
}
catch (InvalidDataException ex)
{
Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex);
manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
bytes = null;
}
}
if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder))
{
manifest = Reidentify(manifest);
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
bytes = null;
}
if (reconcile || File.Exists(sequence.IngestJournalPath))
{
var before = manifest;
manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken),
cancellationToken).ConfigureAwait(false);
if (!ReferenceEquals(before, manifest)) bytes = null;
}
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
locations.Set(manifest.Id, folder.RelativeFolder);
return bytes;
}
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
{
var known = locations.AtFolder(sequence.RelativeFolder);
var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
sequence, cancellationToken).ConfigureAwait(false);
Directory.CreateDirectory(sequence.MetaDirectory);
// Another process may adopt the same folder at the same moment: the first sidecar wins.
return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
}
/// <summary>Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos.</summary>
private async Task<SequenceManifest> ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
{
File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
}
private async Task<SequenceManifest> WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken)
{
var known = new Dictionary<string, Guid>(StringComparer.Ordinal);
var prefix = sequence.ArchiveRelative(string.Empty);
for (var after = -1; ;)
{
var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false);
foreach (var f in page)
{
if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id;
}
if (page.Count < 1000) break;
after = page[^1].Index;
}
if (known.Count == 0) return manifest;
return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() };
}
/// <summary>A copied sequence folder carries the ids of its source; the copy gets ids of its own.</summary>
private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with
{
Id = Guid.CreateVersion7(),
Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(),
SessionKeys = [],
UpdatedAt = DateTimeOffset.UtcNow,
};
/// <summary>Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched).</summary>
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
CancellationToken cancellationToken)
{
ArchiveVolume.EnsureWritable(paths);
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
var removed = 0;
foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
{
if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue;
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
}
return removed;
}
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
private sealed class Locations
{
private readonly Dictionary<Guid, string> _byId = [];
private readonly Dictionary<string, Guid> _byFolder = new(StringComparer.Ordinal);
public Locations(IEnumerable<SequenceLocation> rows)
{
foreach (var row in rows) Set(row.Id, row.Folder);
}
public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null;
/// <summary>The id is indexed at another folder whose sidecar still carries it: this folder is a copy.</summary>
public bool IsCopy(ArchivePaths paths, Guid id, string folder)
{
if (!_byId.TryGetValue(id, out var other) || other == folder) return false;
var otherSidecar = paths.Sequence(other).ManifestPath;
try
{
return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id;
}
catch (InvalidDataException)
{
return false;
}
}
public void Set(Guid id, string folder)
{
if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous);
_byId[id] = folder;
_byFolder[folder] = id;
}
}
}
+344
View File
@@ -0,0 +1,344 @@
using System.Collections.Concurrent;
using System.Globalization;
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Core.Naming;
namespace Titano.Data.Archiving;
/// <summary>A verified staged file and how to name it.</summary>
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
DateTime? ClientCapturedAt, string? PlannedName);
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy);
/// <summary>
/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
/// SHA-256 the sequence already holds is reported as a duplicate and left in staging.
/// </summary>
public sealed class ArchiveIngest : IArchiveIngest
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
private readonly ConcurrentDictionary<Guid, CommitQueue> _queues = new();
private readonly SemaphoreSlim _creation = new(1, 1);
internal ArchiveIngest(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(request);
if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1)
throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request));
var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request));
var staged = new FileInfo(request.StagedPath);
if (!staged.Exists || staged.Length != request.Size)
throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath);
var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256,
MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null);
Guid sequenceId;
if (request.SequenceId is { } id)
{
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
sequenceId = id;
}
else
{
// A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one.
var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}";
sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()),
frame, cancellationToken).ConfigureAwait(false);
}
return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0];
}
/// <summary>Commits files of one import session into the sequence of its key (created on first use).</summary>
internal async Task<IReadOnlyList<IngestResult>> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList<StagedFrame> frames,
CancellationToken cancellationToken)
{
if (frames.Count == 0) return [];
var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false);
return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false);
}
private async Task<Guid> FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
{
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found;
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again;
return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false);
}
finally
{
_creation.Release();
}
}
private async Task<Guid> CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var now = DateTimeOffset.UtcNow;
var m = first.Metadata;
var manifest = new SequenceManifest
{
Id = Guid.CreateVersion7(),
Title = target.Title,
Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
Lens = m.LensModel,
SessionKeys = [target.Key],
CreatedBy = target.CreatedBy,
Created = now,
UpdatedAt = now,
};
var captured = CaptureTime(first);
var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title));
if (wanted.Length == 0) wanted = "Sequence";
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
var sequence = ClaimFolder(paths, wanted, taken, manifest);
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false);
var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false);
if (winner != manifest.Id)
{
// Another process bound the key first: its sequence wins, ours was never used.
await tx.RollbackAsync(cancellationToken).ConfigureAwait(false);
ReleaseFolder(sequence);
return winner;
}
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
return manifest.Id;
}
/// <summary>The first free variant of <paramref name="wanted"/>, claimed by creating its sidecar (first writer wins).</summary>
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet<string> taken, SequenceManifest manifest)
{
for (var n = 1; ; n++)
{
var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})");
if (taken.Contains(folder)) continue;
var sequence = paths.Sequence(folder);
if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue;
Directory.CreateDirectory(sequence.MetaDirectory);
if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence;
}
}
/// <summary>Removes the empty sequence this process created and lost to a concurrent creator.</summary>
private static void ReleaseFolder(SequencePaths sequence)
{
File.Delete(sequence.ManifestPath);
if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory);
if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder);
}
/// <summary>Group commit: whoever gets the sequence's gate commits everything queued for it.</summary>
private async Task<IngestResult[]> EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList<StagedFrame> frames, CancellationToken cancellationToken)
{
var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue());
var pending = frames.Select(f => new Pending(f)).ToList();
foreach (var p in pending) queue.Items.Enqueue(p);
await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
if (pending.Any(p => !p.Completion.Task.IsCompleted))
{
var batch = new List<Pending>();
while (queue.Items.TryDequeue(out var p)) batch.Add(p);
try
{
await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
foreach (var p in batch) p.Completion.TrySetException(ex);
}
}
}
finally
{
queue.Gate.Release();
}
return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false);
}
private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List<Pending> batch, CancellationToken cancellationToken)
{
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var sequence = paths.Sequence(row.Folder);
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
var before = manifest.Frames.Count;
var known = new Dictionary<string, SidecarFrame>(StringComparer.OrdinalIgnoreCase);
foreach (var f in manifest.Frames)
{
if (f.Sha256 is not null) known.TryAdd(f.Sha256, f);
}
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
var duplicates = new List<(Pending Pending, Guid FrameId)>();
foreach (var p in batch)
{
if (known.TryGetValue(p.Frame.Sha256, out var existing))
{
duplicates.Add((p, existing.Id));
continue;
}
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
known[p.Frame.Sha256] = frame;
accepted.Add((p, frame));
}
var named = Name(archive, sequence, manifest, accepted);
IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList()));
var moved = new List<SidecarFrame>();
foreach (var (p, frame) in accepted)
{
var final = named[frame.Id];
try
{
Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!);
File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false);
moved.Add(final);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
p.Completion.TrySetException(ex);
}
}
var updated = SequenceFrames.Insert(manifest, moved);
if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
IngestJournal.Delete(sequence);
var byId = updated.Frames.ToDictionary(f => f.Id);
if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false);
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
foreach (var (p, frameId) in duplicates)
{
if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
}
}
}
/// <summary>Final file names, in capture order: the plan's name or the template with <c>{n}</c> = index + 1, made unique.</summary>
private static Dictionary<Guid, SidecarFrame> Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
List<(Pending Pending, SidecarFrame Frame)> accepted)
{
var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame));
var incoming = accepted.ToDictionary(a => a.Frame.Id);
// Photos adopted in place stay in place; everything else goes to originals/.
var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/'))
? ArchiveLayout.OriginalsFolder + "/"
: string.Empty;
var taken = new HashSet<string>(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase);
var onDisk = Path.Combine(sequence.Folder, directory);
if (Directory.Exists(onDisk))
{
foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file));
}
var template = PathTemplate.Parse(archive.FileTemplate);
var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null;
var named = new Dictionary<Guid, SidecarFrame>();
foreach (var frame in merged.Frames)
{
if (!incoming.TryGetValue(frame.Id, out var entry)) continue;
var staged = entry.Pending.Frame;
var captured = frame.CapturedAt ?? DateTime.Now;
var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title));
var file = directory + name;
var stem = Path.GetFileNameWithoutExtension(name);
for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}");
named[frame.Id] = entry.Frame with { File = file };
}
return named;
}
/// <summary>Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence.</summary>
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
CancellationToken cancellationToken)
{
var inStep = appended.All(f => f.I >= before)
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
if (!inStep)
{
await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
return;
}
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
}
private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) =>
new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate);
private static DateTime CaptureTime(StagedFrame frame) =>
frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client
? client
: frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now;
private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new()
{
CapturedAt = captured,
SessionStart = sessionStart,
Number = number,
NumberInSession = number,
CameraMake = frame.Metadata.Make,
CameraModel = frame.Metadata.Model,
Lens = frame.Metadata.LensModel,
Iso = frame.Metadata.Iso,
ExposureTime = frame.Metadata.ExposureTime,
FNumber = frame.Metadata.FNumber,
OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName),
Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'),
Title = title,
};
private sealed class CommitQueue
{
public SemaphoreSlim Gate { get; } = new(1, 1);
public ConcurrentQueue<Pending> Items { get; } = new();
}
private sealed class Pending(StagedFrame frame)
{
public StagedFrame Frame { get; } = frame;
public TaskCompletionSource<IngestResult> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
}
}
@@ -0,0 +1,222 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Model;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
/// <summary>Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original.</summary>
public sealed record DuplicateGroup(string Sha256, IReadOnlyList<DuplicateFrame> Frames);
/// <summary>Index and disk out of step (BRIEF §12.5 Orphans).</summary>
/// <param name="Sequences">Per indexed sequence: photos the sidecar does not list and frames whose file is gone.</param>
/// <param name="MissingSequences">Index rows whose folder holds no sidecar any more.</param>
/// <param name="UnindexedFolders">Folders with photos or a sidecar that the index does not know yet (a scan adopts them).</param>
public sealed record OrphanReport(IReadOnlyList<SequenceOrphans> Sequences, IReadOnlyList<SequenceLocation> MissingSequences, IReadOnlyList<string> UnindexedFolders)
{
public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count);
public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count);
}
/// <summary>What a maintenance run removed or changed (or would, in a dry run).</summary>
public sealed record MaintenanceReport(
bool DryRun,
IReadOnlyList<string> StaleLeases,
IReadOnlyList<string> TemporaryFiles,
IReadOnlyList<string> CompactedPacks,
int DuplicateGroups,
int DuplicateFlagChanges,
int UnindexedFiles,
int MissingFiles);
/// <summary>Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched.</summary>
public sealed class ArchiveMaintenance
{
/// <summary>Staged copies older than this belong to no running import.</summary>
private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
/// <summary>Temporary files of atomic writes older than this were left by a crash.</summary>
private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
/// <summary>Packs with more reclaimable bytes than this share are compacted.</summary>
private const double CompactionRatio = 0.3;
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
internal ArchiveMaintenance(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
public async Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
(await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false))
.GroupBy(f => f.Sha256, StringComparer.Ordinal)
.Select(g => new DuplicateGroup(g.Key, g.ToList()))
.ToList();
public async Task<OrphanReport> FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var sequences = new List<SequenceOrphans>();
var missing = new List<SequenceLocation>();
foreach (var location in locations)
{
cancellationToken.ThrowIfCancellationRequested();
var sequence = paths.Sequence(location.Folder);
if (!File.Exists(sequence.ManifestPath))
{
missing.Add(location);
continue;
}
var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath));
if (!orphans.IsEmpty) sequences.Add(orphans);
}
var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal);
var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList();
return new OrphanReport(sequences, missing, unindexed);
}
/// <summary>Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept.</summary>
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var removed = 0;
foreach (var id in sequenceIds)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue;
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
}
return removed;
}
/// <summary>Number of maintenance steps (the job checkpoints after each).</summary>
internal const int StepCount = 4;
/// <summary>Runs the maintenance steps from <paramref name="firstStep"/>: stale leases, leftover temporary files,
/// thumbnail pack compaction, duplicate flags. A dry run only reports.</summary>
public async Task<MaintenanceReport> RunAsync(Archive archive, bool dryRun, int firstStep, Func<int, ValueTask> stepDone,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (!dryRun) ArchiveVolume.EnsureWritable(paths);
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
IReadOnlyList<string> leases = [], temps = [], packs = [];
int groups = 0, flagChanges = 0;
for (var step = firstStep; step < StepCount; step++)
{
cancellationToken.ThrowIfCancellationRequested();
switch (step)
{
case 0:
leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
break;
case 1:
temps = PurgeTemporaryFiles(paths, locations, dryRun);
break;
case 2:
packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
break;
case 3:
(groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false);
break;
}
await stepDone(step + 1).ConfigureAwait(false);
}
var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false);
return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles);
}
/// <summary>Staging copies and atomic-write temporaries left behind by crashes (never originals).</summary>
private static List<string> PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun)
{
var now = DateTime.UtcNow;
var found = new List<string>();
void Collect(string directory, string pattern, TimeSpan age, bool recursive)
{
if (!Directory.Exists(directory)) return;
foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
{
if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
}
}
Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
foreach (var location in locations)
{
var sequence = paths.Sequence(location.Folder);
Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false);
Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false);
}
if (!dryRun)
{
foreach (var file in found)
{
try
{
File.Delete(file);
}
catch (IOException ex)
{
Log.Warning("archive", $"Could not remove {file}", ex);
}
}
}
return found.ConvertAll(paths.ToRelative);
}
private async Task<List<string>> CompactPacksAsync(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun, CancellationToken cancellationToken)
{
var compacted = new List<string>();
foreach (var location in locations)
{
var sequence = paths.Sequence(location.Folder);
long garbage, length;
using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
garbage = reader.GarbageBytes;
length = reader.Length;
}
if (length == 0 || garbage < length * CompactionRatio) continue;
compacted.Add(location.Folder);
if (dryRun) continue;
await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
ThumbPackWriter.Compact(sequence.ThumbsPackPath);
}
return compacted;
}
/// <summary>Sets <see cref="FrameFlags.Duplicate"/> on every copy after the first of each hash and clears it elsewhere.</summary>
private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken)
{
var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId);
var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false))
.ToDictionary(f => f.FrameId, f => f.SequenceId);
var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList();
var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList();
if (dryRun) return (groups.Count, set.Count + clear.Count);
foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value))
{
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
}
return (groups.Count, set.Count + clear.Count);
}
}
+232
View File
@@ -0,0 +1,232 @@
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Import;
using Titano.Core.Model;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving;
/// <summary>
/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create
/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and
/// the job handlers for the long operations (enqueue them with <see cref="ArchiveJobs"/>).
/// </summary>
public sealed class ArchiveService
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null)
{
_db = db;
_writer = new SequenceWriter(options ?? new ArchiveServiceOptions());
Indexer = new ArchiveIndexer(db, _writer);
Ingest = new ArchiveIngest(db, _writer, Indexer);
Verifier = new ArchiveVerifier(db, _writer, Indexer);
Maintenance = new ArchiveMaintenance(db, _writer, Indexer);
Renamer = new MassRenamer(db, _writer, Indexer);
Images = new ArchiveImages(db, _writer, Indexer);
Imports = new ImportService(db, Ingest);
}
public ArchiveIndexer Indexer { get; }
/// <summary>Commits staged files into sequences (server uploads, client card imports).</summary>
public ArchiveIngest Ingest { get; }
public ArchiveVerifier Verifier { get; }
public ArchiveMaintenance Maintenance { get; }
public MassRenamer Renamer { get; }
public ArchiveImages Images { get; }
public ImportService Imports { get; }
/// <summary>Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the <see cref="JobEngine"/>.</summary>
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
[
new ScanJobHandler(_db, Indexer, JobKind.Scan),
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
new VerifyJobHandler(_db, Verifier),
new ImportJobHandler(Imports),
new ImageJobHandler(_db, Images, JobKind.Thumbs),
new ImageJobHandler(_db, Images, JobKind.Proxies),
new MaintenanceJobHandler(_db, Maintenance),
];
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
/// <exception cref="InvalidOperationException">The folder already is an archive (open it instead).</exception>
public async Task<Archive> CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(root);
if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive.");
Directory.CreateDirectory(paths.MetaDirectory);
var manifest = new ArchiveManifest
{
Id = Guid.CreateVersion7(),
Name = name,
Created = DateTimeOffset.UtcNow,
LayoutTemplates = templates ?? new LayoutTemplates(),
ProxyPolicy = proxies ?? new ProxyPolicy(),
};
ManifestStore.WriteArchive(paths.ManifestPath, manifest);
return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash
/// interrupted. An offline archive is registered as it was last seen.
/// </summary>
public async Task<Archive> OpenAsync(string root, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(root);
var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false);
if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive;
try
{
await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
{
// The journal stays; the next open tries again. The archive itself is usable meanwhile.
Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex);
}
return archive;
}
/// <summary>
/// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue
/// <see cref="ArchiveJobs.Scan"/> with a full pass to write sidecars and index it.
/// </summary>
public Task<Archive> AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(root);
return File.Exists(paths.ManifestPath)
? OpenAsync(root, cancellationToken)
: CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken);
}
public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath));
/// <summary>Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1).</summary>
public async Task<Sequence> SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList<string> tags, int rating, string? notes,
CancellationToken cancellationToken = default)
{
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with
{
Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(),
Tags = tags,
Rating = rating,
Notes = notes,
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(sequence.Folder);
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
return updated;
}
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
Maintenance.FindDuplicatesAsync(archiveId, cancellationToken);
public async Task<OrphanReport> FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
/// <summary>Orphans fix: index the unlisted photos of a sequence and flag its missing files.</summary>
public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
{
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
}
/// <summary>Orphans fix: drop index rows whose sidecar is gone.</summary>
public async Task<int> RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default) =>
await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken)
.ConfigureAwait(false);
/// <summary>Dry run of a mass rename.</summary>
public async Task<RenamePlan> PlanRenameAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
CancellationToken cancellationToken = default) =>
await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate,
cancellationToken).ConfigureAwait(false);
public async Task ApplyRenameAsync(RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken)
.ConfigureAwait(false);
/// <summary>Plans a card or folder import into an archive (nothing is copied).</summary>
public async Task<ImportPlan> PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress<int>? progress = null,
CancellationToken cancellationToken = default) =>
await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress,
cancellationToken).ConfigureAwait(false);
/// <summary>Dry run of "delete imported" for a card; pass the plan to <see cref="CardEraser.Execute"/> after the user confirms.</summary>
public async Task<ErasePlan> PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
CancellationToken cancellationToken = default) =>
await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress,
cancellationToken).ConfigureAwait(false);
private async Task<Archive> RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken)
{
var archive = new Archive
{
Id = manifest.Id,
Name = manifest.Name,
RootPath = paths.Root,
FolderTemplate = manifest.LayoutTemplates.Folder,
FileTemplate = manifest.LayoutTemplates.File,
SchemaVersion = manifest.SchemaVersion,
Created = manifest.Created,
ProxyLongEdge = manifest.ProxyPolicy.LongEdge,
};
await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false);
return archive;
}
private async Task<Archive> LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) =>
await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId));
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
}
}
/// <summary>Builds the archive jobs; enqueue with <see cref="EnqueueAsync"/>.</summary>
public static class ArchiveJobs
{
public static ArchiveJobRequest Scan(Archive archive, bool full = false) =>
new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User);
public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User);
public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList<Guid>? sequenceIds = null) =>
new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User);
public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) =>
new(JobKind.Import, [archive.Id], archive.Name,
Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User);
public static ArchiveJobRequest Thumbnails(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
public static ArchiveJobRequest Proxies(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) =>
new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters),
JobPriority.Normal);
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
private static string Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo) => JsonSerializer.Serialize(value, typeInfo);
}
@@ -0,0 +1,127 @@
using System.Diagnostics;
using Titano.Core.Archiving;
using Titano.Core.Model;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
/// <summary>A frame whose original is missing, changed or damaged.</summary>
public sealed record VerifyProblem(Guid SequenceId, Guid FrameId, string RelativePath, FrameHealth Health);
/// <summary>Where a verify stands: the next sequence (ordinal in the run) and the next frame index in it.</summary>
public readonly record struct VerifyPosition(int Sequence, int Frame);
/// <summary>Outcome of one verify run (frames before a resume point are counted by the run that checked them).</summary>
public sealed record VerifyReport(int Sequences, long Checked, long Ok, long Hashed, IReadOnlyList<VerifyProblem> Problems);
/// <summary>
/// Re-hashes originals against their sidecar SHA-256 (BRIEF §12.5 Verify): missing, changed and corrupt files get
/// <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> in the sidecar and the index, adopted files get
/// their first hash, and a clean sequence is stamped verified. Progress is flushed in batches so a resumed run
/// continues where the last flush ended.
/// </summary>
public sealed class ArchiveVerifier
{
private const int FlushFrames = 512;
private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30);
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
internal ArchiveVerifier(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
/// <param name="sequenceIds">Sequences to verify; null = the whole archive (ordered by folder).</param>
/// <param name="checkpoint">Called after each flush with the resume position, bytes done and bytes total.</param>
/// <param name="frameDone">Called after each frame with bytes done and bytes total (throughput and ETA).</param>
public async Task<VerifyReport> VerifyAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, VerifyPosition resume,
Func<VerifyPosition, long, long, ValueTask> checkpoint, Func<long, long, ValueTask>? frameDone = null, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false);
var bytesTotal = targets.Sum(t => t.TotalBytes);
var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes);
long checkedFrames = 0, ok = 0, hashed = 0;
var problems = new List<VerifyProblem>();
for (var s = resume.Sequence; s < targets.Count; s++)
{
var target = targets[s];
var sequence = paths.Sequence(target.Folder);
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
var pending = new List<FrameCheck>();
var clock = Stopwatch.StartNew();
var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0;
bytesDone += manifest.Frames.Take(first).Sum(f => f.Size);
for (var i = first; i < manifest.Frames.Count; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false);
checkedFrames++;
if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false);
if (check.Health == FrameHealth.Ok) ok++;
else if (check.Health == FrameHealth.Hashed) hashed++;
else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health));
if (check.ChangesRecord) pending.Add(check);
if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval))
{
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
await checkpoint(new VerifyPosition(s, i + 1), bytesDone, bytesTotal).ConfigureAwait(false);
clock.Restart();
}
}
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
await StampAsync(archive, paths, target, cancellationToken).ConfigureAwait(false);
await checkpoint(new VerifyPosition(s + 1, 0), bytesDone, bytesTotal).ConfigureAwait(false);
}
return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems);
}
private async Task<List<Sequence>> TargetsAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
{
var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList();
var targets = new List<Sequence>(ids.Count);
foreach (var id in ids)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s);
}
return targets;
}
/// <summary>Writes the flags and new hashes of checked frames to the sidecar and the index.</summary>
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
{
if (pending.Count == 0) return;
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
pending.Clear();
}
/// <summary>Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.</summary>
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
{
var sequence = paths.Sequence(target.Folder);
var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
{
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
return m with
{
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
VerifiedAt = DateTimeOffset.UtcNow,
UpdatedAt = DateTimeOffset.UtcNow,
};
}, cancellationToken).ConfigureAwait(false);
var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
}
}
+219
View File
@@ -0,0 +1,219 @@
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Import;
using Titano.Core.Metadata;
using Titano.Core.Model;
using Titano.Core.Naming;
using Titano.Data.Archiving.Jobs;
using Titano.Data.Repositories;
namespace Titano.Data.Archiving;
/// <summary>A card volume as the client sees it (BRIEF §13): identity <c>{serial}-{label}-{capacity}</c>.</summary>
public sealed record CardVolume(string Identity, uint SerialNumber, string? Label, long Capacity);
public enum AfterImport
{
Keep,
/// <summary>Delete from the card the photos whose card file and archive copy both re-hash to the imported hash.</summary>
DeleteImported,
/// <summary>Offer the operating system's format dialog (only after a fully verified import).</summary>
FormatCard,
}
/// <summary>Import defaults remembered per card (<see cref="MediaCard.ProfileJson"/>, BRIEF §12.4).</summary>
public sealed record CardProfile
{
public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
public string FileTemplate { get; init; } = TemplateDefaults.File;
public int SessionGapMinutes { get; init; } = 45;
public int MinimumSessionFrames { get; init; } = 8;
public bool IncludeSubfolders { get; init; } = true;
public bool VerifyCopies { get; init; } = true;
public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
public AfterImport AfterImport { get; init; } = AfterImport.Keep;
public bool Eject { get; init; } = true;
public bool DeepAnalysis { get; init; } = true;
public ImportOptions ToOptions() => new()
{
FolderTemplate = FolderTemplate,
FileTemplate = FileTemplate,
SessionGap = TimeSpan.FromMinutes(SessionGapMinutes),
MinimumSessionFrames = MinimumSessionFrames,
IncludeSubfolders = IncludeSubfolders,
VerifyCopies = VerifyCopies,
Duplicates = Duplicates,
};
}
/// <summary>
/// Card import over the archive index (BRIEF §12.4, §13): card profiles, planning with the archive's history and
/// duplicates, execution into the archive through <see cref="ArchiveIngest"/>, and the verified card erase.
/// </summary>
public sealed class ImportService
{
private readonly TitanoDatabase _db;
private readonly ArchiveIngest _ingest;
internal ImportService(TitanoDatabase db, ArchiveIngest ingest)
{
_db = db;
_ingest = ingest;
}
/// <summary>The card with this identity, registered on first sight (the UI then asks for a nickname).</summary>
public async Task<MediaCard> RegisterCardAsync(CardVolume volume, CancellationToken cancellationToken = default)
{
var now = DateTimeOffset.UtcNow;
if (await _db.MediaCards.FindByIdentityAsync(volume.Identity, cancellationToken).ConfigureAwait(false) is { } known)
{
await _db.MediaCards.TouchAsync(known.Id, now, cancellationToken).ConfigureAwait(false);
return known with { LastSeen = now };
}
var card = new MediaCard
{
Id = Guid.CreateVersion7(),
Identity = volume.Identity,
VolumeSerial = volume.SerialNumber,
Label = volume.Label,
Capacity = volume.Capacity,
FirstSeen = now,
LastSeen = now,
};
await _db.MediaCards.UpsertAsync(card, cancellationToken).ConfigureAwait(false);
return card;
}
public static CardProfile ProfileOf(MediaCard card) =>
card.ProfileJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.CardProfile) ?? new() : new();
/// <summary>Stores the nickname, defaults and auto-import choice of a card.</summary>
public async Task<MediaCard> SaveCardAsync(Guid cardId, string? nickname, CardProfile profile, bool autoImport, CancellationToken cancellationToken = default)
{
var card = await _db.MediaCards.GetAsync(cardId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown card {cardId}.", nameof(cardId));
var updated = card with
{
Nickname = string.IsNullOrWhiteSpace(nickname) ? null : nickname.Trim(),
ProfileJson = JsonSerializer.Serialize(profile, ArchiveJobJson.Default.CardProfile),
AutoImport = autoImport,
};
await _db.MediaCards.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
return updated;
}
/// <summary>Scans the source, reads light metadata and plans the import into the archive. Nothing is written
/// except the card's camera identity on its first plan.</summary>
/// <param name="progress">Photos read so far.</param>
public async Task<ImportPlan> PlanAsync(Archive archive, string sourceRoot, ImportOptions options, Guid? cardId, IProgress<int>? progress = null,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var problems = new List<ImportProblem>();
var files = CardScanner.Find(sourceRoot, options.IncludeSubfolders, cancellationToken);
var photos = CardScanner.ReadMetadata(sourceRoot, files, problems, progress, cancellationToken);
var card = cardId is { } id ? await _db.MediaCards.GetAsync(id, cancellationToken).ConfigureAwait(false) : null;
if (card is { CameraModel: null } && photos.Count > 0)
{
var first = photos[0].Metadata;
await _db.MediaCards.UpsertAsync(card with { CameraMake = first.Make, CameraModel = first.Model, CameraSerial = first.BodySerial },
cancellationToken).ConfigureAwait(false);
}
var imported = card is null ? new HashSet<ImportKey>() : await _db.MediaCards.GetImportedKeysAsync(card.Id, cancellationToken).ConfigureAwait(false);
var folders = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
var lookup = new IndexLookup(_db, archive.Id, paths, imported, folders);
return await ImportPlanner.PlanAsync(sourceRoot, paths.Root, photos, problems, options, lookup, cancellationToken).ConfigureAwait(false);
}
/// <summary>Runs (or resumes) a confirmed plan; <paramref name="committed"/> sees each batch after its history is recorded.</summary>
internal async Task<ImportProgress> ExecuteAsync(ImportJobParameters job, Guid jobId, int startAt, Func<ImportBatch, ValueTask> committed,
Func<ImportProgress, ValueTask>? progress, CancellationToken cancellationToken)
{
var archive = await _db.Archives.GetAsync(job.ArchiveId, cancellationToken).ConfigureAwait(false)
?? throw new InvalidOperationException($"Unknown archive {job.ArchiveId}.");
ArchiveVolume.EnsureWritable(new ArchivePaths(archive.RootPath));
var sink = new IngestSink(_ingest, archive, job.UserId?.ToString());
return await new ImportExecutor(job.Plan, sink).RunAsync(startAt, job.Verify, async batch =>
{
if (job.CardId is { } card) await RecordHistoryAsync(card, jobId, batch, cancellationToken).ConfigureAwait(false);
await committed(batch).ConfigureAwait(false);
}, progress, cancellationToken).ConfigureAwait(false);
}
/// <summary>Dry run of "delete imported": every card file with a recorded import, checked against its archive copy.</summary>
public async Task<ErasePlan> PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var candidates = new List<EraseCandidate>();
foreach (var record in await _db.MediaCards.ListVerifiableImportsAsync(cardId, cancellationToken).ConfigureAwait(false))
{
if (!File.Exists(Path.Combine(sourceRoot, record.Key.Path))) continue;
if (await _db.Frames.GetAsync(record.FrameId!.Value, cancellationToken).ConfigureAwait(false) is not { } frame) continue;
candidates.Add(new EraseCandidate(record.Key.Path, record.Key.Size, record.Sha256!, paths.FromRelative(frame.RelativePath)));
}
return await CardEraser.PlanAsync(sourceRoot, candidates, progress, cancellationToken).ConfigureAwait(false);
}
private Task RecordHistoryAsync(Guid cardId, Guid jobId, ImportBatch batch, CancellationToken cancellationToken)
{
var now = DateTimeOffset.UtcNow;
var records = new List<ImportRecord>();
foreach (var r in batch.Results)
{
var key = new ImportKey(r.Item.SourcePath, r.Item.Size, new DateTimeOffset(DateTime.SpecifyKind(r.Item.ModifiedUtc, DateTimeKind.Utc)));
switch (r.Status)
{
case ImportItemStatus.Copied or ImportItemStatus.Duplicate:
records.Add(new ImportRecord(cardId, key, r.Sha256, r.Commit?.FrameId, jobId, now));
break;
case ImportItemStatus.Skipped when r.Item.Action == ImportAction.SkipInArchive:
records.Add(new ImportRecord(cardId, key, r.Item.Sha256, r.Item.ExistingFrameId, jobId, now));
break;
}
}
return records.Count == 0 ? Task.CompletedTask : _db.MediaCards.RecordImportsAsync(records, cancellationToken);
}
/// <summary>Planner questions answered from the index and the archive folder.</summary>
private sealed class IndexLookup(TitanoDatabase db, Guid archiveId, ArchivePaths paths, IReadOnlySet<ImportKey> imported, HashSet<string> indexedFolders)
: IImportLookup
{
public Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken) =>
Task.FromResult(imported.Contains(new ImportKey(file.RelativePath, file.Size, new DateTimeOffset(DateTime.SpecifyKind(file.ModifiedUtc, DateTimeKind.Utc)))));
public async Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) =>
(await db.Frames.FindBySizeAndCaptureAsync(archiveId, size, capturedAt, cancellationToken).ConfigureAwait(false))
.Select(f => new ArchiveMatch(f.Id, f.Sha256)).ToList();
public async Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) =>
await db.Sequences.FindBySessionKeyAsync(archiveId, sessionKey, cancellationToken).ConfigureAwait(false) is { } id
? (await db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder
: null;
public bool IsFolderTaken(string folder)
{
if (indexedFolders.Contains(folder)) return true;
var path = paths.Sequence(folder).Folder;
return Directory.Exists(path) && Directory.EnumerateFileSystemEntries(path).Any();
}
}
/// <summary>Import destination "Archive": each plan session becomes (or continues) the sequence of its session key.</summary>
private sealed class IngestSink(ArchiveIngest ingest, Archive archive, string? createdBy) : IImportSink
{
public string StagingDirectory { get; } = new ArchivePaths(archive.RootPath).StagingDirectory;
public async Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken)
{
var frames = files.Select(f => new StagedFrame(f.StagedPath, Path.GetFileName(f.Item.SourcePath), f.Item.Size, f.Sha256,
MetadataReader.Read(f.StagedPath, cancellationToken), f.Item.CapturedAt, f.Item.FileName)).ToList();
var results = await ingest.CommitSessionAsync(archive, new SequenceTarget(session.SessionKey, session.Title, session.Folder, createdBy),
frames, cancellationToken).ConfigureAwait(false);
return results.Select(r => new SinkCommit(r.FrameId, r.RelativePath, r.Duplicate)).ToList();
}
}
}
@@ -0,0 +1,38 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Titano.Core.Import;
using Titano.Core.Model;
namespace Titano.Data.Archiving.Jobs;
/// <summary><see cref="JobKind.Scan"/>: target = the archive.</summary>
public sealed record ScanParameters(bool Full);
/// <summary><see cref="JobKind.Verify"/>: target = the archive; null sequences = all of them.</summary>
public sealed record VerifyParameters(IReadOnlyList<Guid>? SequenceIds);
/// <summary><see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/>: targets = sequences.</summary>
public sealed record ImageJobParameters(bool Force);
/// <summary><see cref="JobKind.Maintenance"/>: target = the archive.</summary>
public sealed record MaintenanceParameters(bool DryRun);
/// <summary><see cref="JobKind.Import"/>: the exact plan the user confirmed.</summary>
public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId);
/// <summary>A job ready to enqueue.</summary>
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority);
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
[JsonSerializable(typeof(ScanParameters))]
[JsonSerializable(typeof(VerifyParameters))]
[JsonSerializable(typeof(ImageJobParameters))]
[JsonSerializable(typeof(MaintenanceParameters))]
[JsonSerializable(typeof(ImportJobParameters))]
[JsonSerializable(typeof(CardProfile))]
internal sealed partial class ArchiveJobJson : JsonSerializerContext
{
/// <summary>Parameters of a job, or <paramref name="fallback"/> when it has none.</summary>
public static T Read<T>(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo, T fallback) =>
job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback;
}
@@ -0,0 +1,51 @@
using System.Globalization;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/> over the target sequences. Progress counts
/// sequences; checkpoint: <c>sequence:frame</c>.
/// </summary>
internal sealed class ImageJobHandler(TitanoDatabase db, ArchiveImages images, JobKind kind) : IJobHandler
{
public JobKind Kind => kind;
public async Task RunAsync(JobContext context)
{
var force = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ImageJobParameters, new ImageJobParameters(false)).Force;
var (first, firstFrame) = Parse(context.Checkpoint);
var targets = context.Job.TargetIds;
for (var s = first; s < targets.Count; s++)
{
context.Cancellation.ThrowIfCancellationRequested();
if (await db.Sequences.GetAsync(targets[s], context.Cancellation).ConfigureAwait(false) is not { } sequence
|| await db.Archives.GetAsync(sequence.ArchiveId, context.Cancellation).ConfigureAwait(false) is not { } archive)
{
context.Log(Loc.L("Archive.Log.SequenceMissing", targets[s]));
continue;
}
var index = s;
ValueTask Checkpoint(int nextFrame, int frames) =>
context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{index}:{nextFrame}"), index, targets.Count);
var start = s == first ? firstFrame : 0;
var result = kind == JobKind.Thumbs
? await images.BuildThumbnailsAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false)
: await images.BuildProxiesAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false);
context.Log(Loc.L("Archive.Log.ImagesDone", sequence.DisplayTitle, result.Built, result.Reused, result.Undecodable));
await context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{s + 1}:0"), s + 1, targets.Count).ConfigureAwait(false);
}
}
private static (int Sequence, int Frame) Parse(string? checkpoint)
{
var parts = checkpoint?.Split(':');
return parts is [var s, var f]
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
? (sequence, frame)
: (0, 0);
}
}
@@ -0,0 +1,33 @@
using System.Globalization;
using System.Text.Json;
using Titano.Core.Import;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.Import"/>: executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the
/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion
/// continues there.
/// </summary>
internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
{
public JobKind Kind => JobKind.Import;
public async Task RunAsync(JobContext context)
{
var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ImportJobParameters) : null)
?? throw new InvalidOperationException($"Import job {context.Job.Id} has no plan.");
var start = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0;
var final = await imports.ExecuteAsync(parameters, context.Job.Id, start, async batch =>
{
foreach (var failed in batch.Results.Where(r => r.Status == ImportItemStatus.Failed))
context.Log(Loc.L("Archive.Log.ImportFailed", failed.Item.SourcePath, failed.Error));
await context.CheckpointAsync(batch.NextItem.ToString(CultureInfo.InvariantCulture), batch.Progress.BytesDone, batch.Progress.BytesTotal)
.ConfigureAwait(false);
}, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false);
context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed));
}
}
@@ -0,0 +1,29 @@
using System.Globalization;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>
/// <see cref="JobKind.Maintenance"/>: stale leases, leftover temporary files, thumbnail pack compaction, duplicate
/// flags, then an orphan count. Checkpoint: the number of steps done.
/// </summary>
internal sealed class MaintenanceJobHandler(TitanoDatabase db, ArchiveMaintenance maintenance) : IJobHandler
{
public JobKind Kind => JobKind.Maintenance;
public async Task RunAsync(JobContext context)
{
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
var dryRun = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.MaintenanceParameters, new MaintenanceParameters(false)).DryRun;
var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var step) ? step : 0;
var report = await maintenance.RunAsync(archive, dryRun, first,
done => context.CheckpointAsync(done.ToString(CultureInfo.InvariantCulture), done, ArchiveMaintenance.StepCount),
context.Cancellation).ConfigureAwait(false);
if (dryRun) context.Log(Loc.L("Archive.Log.DryRun"));
context.Log(Loc.L("Archive.Log.MaintenanceDone", report.StaleLeases.Count, report.TemporaryFiles.Count, report.CompactedPacks.Count,
report.DuplicateFlagChanges));
context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles));
}
}
@@ -0,0 +1,41 @@
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary>Resolves the archive a job targets (its first target id).</summary>
internal static class JobArchive
{
public static async Task<Archive> LoadAsync(TitanoDatabase db, Job job, CancellationToken cancellationToken) =>
job.TargetIds.Count > 0 && await db.Archives.GetAsync(job.TargetIds[0], cancellationToken).ConfigureAwait(false) is { } archive
? archive
: throw new InvalidOperationException($"{job.Kind} job {job.Id} does not target a known archive.");
}
/// <summary>
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:&lt;folder&gt;</c>, the last folder indexed.
/// </summary>
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
{
private const string Prefix = "after:";
public JobKind Kind => kind;
public async Task RunAsync(JobContext context)
{
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
var mode = kind == JobKind.Rebuild ? ScanMode.Rebuild
: ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).Full ? ScanMode.Full
: ScanMode.Incremental;
var resume = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) ? checkpoint[Prefix.Length..] : null;
var summary = await indexer.ScanAsync(archive, mode, resume, async p =>
{
if (p.Changed) await context.CheckpointAsync(Prefix + p.Folder, p.FoldersDone, p.FoldersTotal).ConfigureAwait(false);
else await context.ReportAsync(p.FoldersDone, p.FoldersTotal, p.Folder).ConfigureAwait(false);
}, context.Cancellation).ConfigureAwait(false);
foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error));
context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed));
}
}
@@ -0,0 +1,47 @@
using System.Globalization;
using Titano.Core.Archiving;
using Titano.Core.Localization;
using Titano.Core.Model;
using Titano.Data.Jobs;
namespace Titano.Data.Archiving.Jobs;
/// <summary><see cref="JobKind.Verify"/>; progress in bytes hashed. Checkpoint: <c>sequence:frame</c>.</summary>
internal sealed class VerifyJobHandler(TitanoDatabase db, ArchiveVerifier verifier) : IJobHandler
{
/// <summary>Problems listed one per log line; the rest are counted (the index flags them all).</summary>
private const int LoggedProblems = 25;
public JobKind Kind => JobKind.Verify;
public async Task RunAsync(JobContext context)
{
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null));
var report = await verifier.VerifyAsync(archive, parameters.SequenceIds, Parse(context.Checkpoint),
(position, done, total) => context.CheckpointAsync(
string.Create(CultureInfo.InvariantCulture, $"{position.Sequence}:{position.Frame}"), done, total),
(done, total) => context.ReportAsync(done, total),
context.Cancellation).ConfigureAwait(false);
foreach (var problem in report.Problems.Take(LoggedProblems))
{
context.Log(problem.Health switch
{
FrameHealth.Missing => Loc.L("Archive.Log.FrameMissing", problem.RelativePath),
FrameHealth.Changed => Loc.L("Archive.Log.FrameChanged", problem.RelativePath),
_ => Loc.L("Archive.Log.FrameCorrupt", problem.RelativePath),
});
}
context.Log(Loc.L("Archive.Log.VerifyDone", report.Checked, report.Ok, report.Hashed, report.Problems.Count));
}
private static VerifyPosition Parse(string? checkpoint)
{
var parts = checkpoint?.Split(':');
return parts is [var s, var f]
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
? new VerifyPosition(sequence, frame)
: default;
}
}
+134
View File
@@ -0,0 +1,134 @@
using Titano.Core.Archiving;
using Titano.Core.Diagnostics;
using Titano.Core.Localization;
using Titano.Core.Model;
namespace Titano.Data.Archiving;
/// <summary>The plan no longer matches the archive (a sequence moved or vanished since it was made).</summary>
public sealed class StaleRenamePlanException(Guid sequenceId)
: InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
{
public Guid SequenceId { get; } = sequenceId;
}
/// <summary>
/// Mass rename of sequence folders and titles (BRIEF §7, §12.5): <see cref="PlanAsync"/> is the dry run;
/// <see cref="ApplyAsync"/> makes exactly the planned changes to folders, sidecars and the index, step by step, with a
/// journal on the archive so an interrupted rename is completed by <see cref="ResumePendingAsync"/>.
/// </summary>
public sealed class MassRenamer
{
private readonly TitanoDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
internal MassRenamer(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
/// <param name="folderTemplate">New folder template (<c>{title}</c> = the new title), or null to keep folders.</param>
/// <param name="titleTemplate">New title template (<c>{title}</c> = the current title), or null to keep titles.</param>
public async Task<RenamePlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var subjects = new List<RenameSubject>(sequenceIds.Count);
foreach (var id in sequenceIds)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id)
throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds));
subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens));
}
return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder),
folder => Directory.Exists(paths.Sequence(folder).Folder));
}
/// <summary>Applies a plan; each completed step is journaled before the next starts.</summary>
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it; nothing of that step was changed.</exception>
public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken);
/// <summary>Completes renames interrupted by a crash; returns how many plans were resumed.</summary>
public async Task<int> ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
var pending = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList();
foreach (var journal in pending) await RunAsync(archive, journal, resuming: true, null, cancellationToken).ConfigureAwait(false);
return pending.Count;
}
/// <param name="resuming">Steps whose sequence the index no longer knows are dropped instead of failing the rename:
/// their sidecars still carry the ids, so the next scan indexes them wherever they ended up.</param>
private async Task RunAsync(Archive archive, RenameJournal journal, bool resuming, IProgress<int>? progress, CancellationToken cancellationToken)
{
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal));
var completed = journal.Completed.ToHashSet();
if (!resuming)
{
// A plan made before another change is refused before anything moves.
foreach (var step in journal.Plan.Steps)
{
if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder)
throw new StaleRenamePlanException(step.SequenceId);
}
}
RenameJournal.Write(paths, journal);
for (var i = 0; i < journal.Plan.Steps.Count; i++)
{
var step = journal.Plan.Steps[i];
if (completed.Contains(step.SequenceId)) continue;
cancellationToken.ThrowIfCancellationRequested();
try
{
await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
}
catch (StaleRenamePlanException) when (resuming)
{
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
}
completed.Add(step.SequenceId);
RenameJournal.Write(paths, journal with { Completed = [.. completed] });
progress?.Report(i + 1);
}
RenameJournal.Delete(paths, journal.Plan.Id);
}
/// <summary>Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it.</summary>
private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
{
await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false))
{
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId);
RenamePlanner.MoveFolder(paths, step);
var sequence = paths.Sequence(step.ToFolder);
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
if (manifest.Title != step.ToTitle)
{
manifest = manifest with
{
Title = step.ToTitle,
Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
UpdatedAt = DateTimeOffset.UtcNow,
};
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
}
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
}
}
}
@@ -0,0 +1,106 @@
using System.Text.Json;
using Titano.Core.Archiving;
using Titano.Core.Model;
namespace Titano.Data.Archiving;
/// <summary>Sidecar records to index rows. Fields the sidecar does not carry are kept from the existing row.</summary>
internal static class SequenceMapping
{
public static Frame ToFrame(SidecarFrame f, Guid sequenceId, SequencePaths sequence) => new()
{
Id = f.Id,
SequenceId = sequenceId,
Index = f.I,
RelativePath = sequence.ArchiveRelative(f.File),
FileName = Path.GetFileName(f.File),
OriginalFileName = f.Original,
Size = f.Size,
Sha256 = f.Sha256,
CapturedAt = f.CapturedAt,
UtcOffsetMinutes = f.UtcOffset,
TimestampSource = f.TimeSource,
ExposureTime = f.Exposure,
FNumber = f.FNumber,
Iso = f.Iso,
FocalLength = f.Focal,
ExposureBias = f.Bias,
WhiteBalance = f.Wb,
Orientation = f.Orientation,
Width = f.W,
Height = f.H,
Format = f.Format,
PreviewOffset = f.PreviewOffset,
PreviewLength = f.PreviewLength,
Flags = f.Flags,
};
public static List<Frame> ToFrames(SequenceManifest manifest, SequencePaths sequence) =>
manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList();
/// <param name="existing">The current row; its render, analysis, sync and ownership fields are kept, and so are its
/// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only).</param>
public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing)
{
var indexAhead = existing is not null && existing.SettingsVersion > m.Versions.Settings;
var times = m.Frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList();
var flagged = m.Frames.Any(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0);
return new Sequence
{
Id = m.Id,
ArchiveId = archiveId,
Title = indexAhead ? existing!.Title : m.Title,
Folder = sequence.RelativeFolder,
CameraMake = m.Camera?.Make,
CameraModel = m.Camera?.Model,
CameraSerial = m.Camera?.Serial,
Lens = m.Lens,
FirstCapture = times.Count > 0 ? times[0] : null,
LastCapture = times.Count > 0 ? times[^1] : null,
FrameCount = m.Frames.Count,
MedianInterval = MedianInterval(times),
TotalBytes = m.Frames.Sum(f => f.Size),
Status = flagged ? m.Status | SequenceStatus.HasErrors : m.Status & ~SequenceStatus.HasErrors,
VerifiedAt = m.VerifiedAt,
Tags = indexAhead ? existing!.Tags : m.Tags,
Rating = indexAhead ? existing!.Rating : m.Rating,
Notes = indexAhead ? existing!.Notes : m.Notes,
SettingsVersion = indexAhead ? existing!.SettingsVersion : m.Versions.Settings,
FramesVersion = m.Versions.Frames,
UpdatedAt = m.UpdatedAt,
RenderedCount = existing?.RenderedCount ?? 0,
HeroFrameId = existing?.HeroFrameId,
SettingsJson = existing?.SettingsJson ?? ReadProject(sequence),
AnalysisJson = existing?.AnalysisJson,
Origin = existing?.Origin ?? SequenceOrigin.Local,
SyncState = existing?.SyncState ?? SyncState.LocalOnly,
OwnerUserId = existing?.OwnerUserId ?? (Guid.TryParse(m.CreatedBy, out var owner) ? owner : null),
WarningCount = existing?.WarningCount ?? 0,
};
}
private static double? MedianInterval(List<DateTime> times)
{
if (times.Count < 2) return null;
var gaps = new double[times.Count - 1];
for (var i = 1; i < times.Count; i++) gaps[i - 1] = (times[i] - times[i - 1]).TotalSeconds;
Array.Sort(gaps);
var mid = gaps.Length / 2;
return gaps.Length % 2 == 1 ? gaps[mid] : (gaps[mid - 1] + gaps[mid]) / 2;
}
/// <summary>The project file is owned by the editor; a rebuilt index takes its text as-is when it is valid JSON.</summary>
private static string? ReadProject(SequencePaths sequence)
{
if (!File.Exists(sequence.ProjectPath)) return null;
var text = File.ReadAllText(sequence.ProjectPath);
try
{
using (JsonDocument.Parse(text)) return text;
}
catch (JsonException)
{
return null;
}
}
}
@@ -0,0 +1,92 @@
using System.Collections.Concurrent;
using Titano.Core.Archiving;
using Titano.Core.Imaging;
namespace Titano.Data.Archiving;
/// <summary>Settings shared by the archive services.</summary>
public sealed record ArchiveServiceOptions
{
/// <summary>Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows).</summary>
public ImageDecoder Decoder { get; init; } = new([]);
/// <summary>Name written into lease files so a waiting writer can say who holds a sequence.</summary>
public string Owner { get; init; } = "titano@" + Environment.MachineName;
/// <summary>How long a writer waits for a sequence held by another process before giving up.</summary>
public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
}
/// <summary>
/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease
/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar,
/// with an interrupted ingest batch completed first.
/// </summary>
internal sealed class SequenceWriter(ArchiveServiceOptions options)
{
private readonly ConcurrentDictionary<Guid, SemaphoreSlim> _gates = new();
public ArchiveServiceOptions Options => options;
/// <summary>Holds the sequence exclusively until disposed.</summary>
public async Task<IAsyncDisposable> LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken)
{
var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1));
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
{
var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait,
options.LeaseDuration, cancellationToken).ConfigureAwait(false);
return new Held(gate, lease);
}
catch
{
gate.Release();
throw;
}
}
/// <summary>
/// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first.
/// </summary>
/// <exception cref="InvalidDataException">The sidecar in the folder belongs to another sequence.</exception>
public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId)
{
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
if (manifest.Id != sequenceId)
throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
{
ManifestStore.WriteSequence(sequence.ManifestPath, recovered);
manifest = recovered;
}
IngestJournal.Delete(sequence);
return manifest;
}
/// <summary>Applies <paramref name="edit"/> to the sidecar under the lock; a null result writes nothing.</summary>
public async Task<SequenceManifest> UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder,
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
{
var sequence = archive.Sequence(folder);
await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false))
{
var manifest = ReadLocked(sequence, sequenceId);
if (edit(manifest) is not { } updated) return manifest;
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
return updated;
}
}
private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable
{
public ValueTask DisposeAsync()
{
lease.Dispose();
gate.Release();
return ValueTask.CompletedTask;
}
}
}
@@ -0,0 +1,23 @@
-- Archive index state (BRIEF §7, §14.4). Mirrors sqlite/0002_archive_index.sql.
-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
CREATE TABLE sequence_sessions (
archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
session_key text NOT NULL,
sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
PRIMARY KEY (archive_id, session_key)
);
CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
-- Times are UTC ticks (100 ns), compared exactly with the file system's.
CREATE TABLE archive_folders (
archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
folder text COLLATE "C" NOT NULL,
sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL,
sidecar_size bigint,
sidecar_ticks bigint,
photos_ticks bigint,
scanned_at timestamptz NOT NULL,
PRIMARY KEY (archive_id, folder)
);
@@ -0,0 +1,23 @@
-- Archive index state (BRIEF §7, §14.4). Mirrors postgres/0002_archive_index.sql.
-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
CREATE TABLE sequence_sessions (
archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
session_key TEXT NOT NULL,
sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
PRIMARY KEY (archive_id, session_key)
);
CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
-- Times are UTC ticks (100 ns), compared exactly with the file system's.
CREATE TABLE archive_folders (
archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
folder TEXT NOT NULL,
sequence_id TEXT REFERENCES sequences (id) ON DELETE SET NULL,
sidecar_size INTEGER,
sidecar_ticks INTEGER,
photos_ticks INTEGER,
scanned_at TEXT NOT NULL,
PRIMARY KEY (archive_id, folder)
);
@@ -1,7 +1,11 @@
using Titano.Core.Archiving;
using Titano.Core.Model;
namespace Titano.Data.Repositories;
/// <summary>What the last scan saw in one sequence folder (sidecar and photo-directory stamps).</summary>
public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? Sidecar, FileStamp? Photos);
/// <summary>Archive roots (BRIEF §5, §7).</summary>
public sealed class ArchiveRepository
{
@@ -10,6 +14,9 @@ public sealed class ArchiveRepository
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
private static readonly string FolderStateUpsertSql = SqlDialect.Upsert("archive_folders",
["archive_id", "folder", "sequence_id", "sidecar_size", "sidecar_ticks", "photos_ticks", "scanned_at"], 2);
private readonly DbAccess _db;
internal ArchiveRepository(DbAccess db) => _db = db;
@@ -29,6 +36,31 @@ public sealed class ArchiveRepository
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
/// <summary>Scan stamps of every folder of an archive, by folder.</summary>
public async Task<IReadOnlyDictionary<string, FolderScanState>> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
{
var rows = await _db.ListAsync(
"SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
cancellationToken).ConfigureAwait(false);
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
}
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync(FolderStateUpsertSql,
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
cancellationToken).ConfigureAwait(false);
public async Task<bool> DeleteFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
await _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1 AND folder = $2", [archiveId, folder], cancellationToken)
.ConfigureAwait(false) > 0;
/// <summary>Forgets every scan stamp of an archive, so the next scan reads every sidecar (rebuild).</summary>
public Task<int> ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
private static Archive Map(DbRow r) => new()
{
Id = r.GetGuid(0),
@@ -4,6 +4,15 @@ using Titano.Core.Model;
namespace Titano.Data.Repositories;
/// <summary>Integrity fields a verify rewrites: the flags and the hash of a newly hashed file.</summary>
public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256);
/// <summary>A frame and the sequence it belongs to.</summary>
public readonly record struct FrameRef(Guid FrameId, Guid SequenceId);
/// <summary>A frame whose content (SHA-256) also exists elsewhere in the archive.</summary>
public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath);
/// <summary>Light statistics of one frame, written by the analysis job.</summary>
public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh);
@@ -25,6 +34,12 @@ public sealed class FrameRepository
/// <summary>Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row).</summary>
private const int Batch = 1000;
/// <summary>Rows per multi-row upsert: 30 x 27 parameters stays below SQLite's historical 999-variable limit.</summary>
private const int UpsertRows = 30;
/// <summary>Light statistics belong to the analysis job; a sync from the sidecar never clears them.</summary>
private static readonly string[] AnalysisColumns = ["mean_log_luminance", "clip_low", "clip_high"];
private readonly DbAccess _db;
internal FrameRepository(DbAccess db) => _db = db;
@@ -112,6 +127,89 @@ public sealed class FrameRepository
return updated;
}, cancellationToken);
/// <summary>
/// Makes the frames of a sequence exactly <paramref name="frames"/> (from its sidecar): rows are upserted by id,
/// rows not listed are deleted, light statistics of kept rows survive. Indexes may be renumbered freely.
/// </summary>
public Task<int> SyncSequenceAsync(Guid sequenceId, IReadOnlyList<Frame> frames, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
// Park current rows on negative indexes so renumbered rows never collide on (sequence_id, idx).
await session.ExecuteAsync("UPDATE frames SET idx = -idx - 1 WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
var updates = string.Join(", ", ColumnNames.Skip(1).Where(c => !AnalysisColumns.Contains(c)).Select(c => $"{c} = excluded.{c}"));
for (var start = 0; start < frames.Count; start += UpsertRows)
{
var count = Math.Min(UpsertRows, frames.Count - start);
var sql = new StringBuilder("INSERT INTO frames (").Append(Columns).Append(") VALUES ");
var args = new DbParam[count * ColumnNames.Length];
for (var i = 0; i < count; i++)
{
var values = Values(frames[start + i]);
sql.Append(i > 0 ? ", (" : "(");
for (var c = 0; c < values.Length; c++)
{
args[i * values.Length + c] = new DbParam(values[c]);
sql.Append(c > 0 ? ", $" : "$").Append(i * values.Length + c + 1);
}
sql.Append(')');
}
sql.Append(" ON CONFLICT (id) DO UPDATE SET ").Append(updates);
await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
}
return await session.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1 AND idx < 0", [sequenceId], cancellationToken).ConfigureAwait(false);
}, cancellationToken);
/// <summary>Number of frames of a sequence.</summary>
public async Task<int> CountBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
(int)await _db.ScalarAsync<long>("SELECT count(*) FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
/// <summary>Frames of an archive with this size and capture time (card import duplicate candidates).</summary>
public Task<IReadOnlyList<Frame>> FindBySizeAndCaptureAsync(Guid archiveId, long size, DateTime capturedAt, CancellationToken cancellationToken = default) =>
_db.ListAsync(
$"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
"WHERE s.archive_id = $1 AND f.size = $2 AND f.captured_at = $3",
[archiveId, size, SequenceRepository.Wall(capturedAt)], Map, cancellationToken);
/// <summary>Every frame of an archive whose SHA-256 occurs more than once, grouped by hash then sequence and index.</summary>
public Task<IReadOnlyList<DuplicateFrame>> ListDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT f.sha256, f.id, f.sequence_id, f.idx, f.relative_path FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
"WHERE s.archive_id = $1 AND f.sha256 IN (SELECT f2.sha256 FROM frames f2 JOIN sequences s2 ON s2.id = f2.sequence_id " +
"WHERE s2.archive_id = $1 AND f2.sha256 IS NOT NULL GROUP BY f2.sha256 HAVING count(*) > 1) " +
"ORDER BY f.sha256, s.first_capture, f.sequence_id, f.idx",
[archiveId], r => new DuplicateFrame(r.GetString(0), r.GetGuid(1), r.GetGuid(2), r.GetInt32(3), r.GetString(4)), cancellationToken);
/// <summary>Frames of an archive that carry every bit of <paramref name="flags"/>.</summary>
public Task<IReadOnlyList<FrameRef>> ListFlaggedAsync(Guid archiveId, FrameFlags flags, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT f.id, f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 AND (f.flags & $2) = $2",
[archiveId, (int)flags], r => new FrameRef(r.GetGuid(0), r.GetGuid(1)), cancellationToken);
/// <summary>Rewrites the relative paths of a sequence's frames after its folder moved (mass rename).</summary>
/// <remarks>Only paths still under <paramref name="oldFolder"/> change, so repeating it after a crash is harmless.</remarks>
public Task<int> RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default)
{
// substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder.
var prefix = oldFolder + "/";
var length = prefix.EnumerateRunes().Count();
return _db.ExecuteAsync(
"UPDATE frames SET relative_path = $2 || substr(relative_path, $3) WHERE sequence_id = $1 AND substr(relative_path, 1, $4) = $5",
[sequenceId, newFolder, length, length, prefix], cancellationToken);
}
/// <summary>Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated.</summary>
public Task<int> UpdateIntegrityAsync(IReadOnlyList<FrameIntegrity> frames, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var updated = 0;
foreach (var f in frames)
{
updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
[f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false);
}
return updated;
}, cancellationToken);
/// <summary>Removes every frame of a sequence; returns how many.</summary>
public Task<int> DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken);
@@ -96,6 +96,16 @@ public sealed class MediaCardRepository
await _db.ScalarAsync<int?>("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4",
[cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null;
/// <summary>Every recorded import from a card that has a hash and a frame (candidates for a verified card erase).</summary>
public Task<IReadOnlyList<ImportRecord>> ListVerifiableImportsAsync(Guid cardId, CancellationToken cancellationToken = default) =>
_db.ListAsync(
"SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history " +
"WHERE card_id = $1 AND sha256 IS NOT NULL AND frame_id IS NOT NULL ORDER BY path",
[cardId],
r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)),
r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)),
cancellationToken);
/// <summary>Most recent imports from a card.</summary>
public Task<IReadOnlyList<ImportRecord>> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) =>
_db.ListAsync(
@@ -79,6 +79,9 @@ public sealed record ArchiveHealth(
long CorruptFrames,
long DuplicateFrames);
/// <summary>Where a sequence lives in its archive.</summary>
public sealed record SequenceLocation(Guid Id, string Folder, string? Title);
/// <summary>Sequences (projects) with their tags; Library listing and Archive statistics.</summary>
public sealed class SequenceRepository
{
@@ -259,6 +262,45 @@ public sealed class SequenceRepository
}, cancellationToken);
}
/// <summary>Folder and title of every sequence of an archive, by folder.</summary>
public Task<IReadOnlyList<SequenceLocation>> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
_db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId],
r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken);
/// <summary>The sequence created for a session key (automatic grouping of uploads and card sessions).</summary>
public Task<Guid?> FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) =>
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken);
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
await session.ExecuteAsync(
"INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING",
[archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false);
return (await session.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
}, cancellationToken);
/// <summary>Moves a sequence to another folder and/or title (mass rename); false when it does not exist.</summary>
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
_db.InTransactionAsync(async session =>
{
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
.ConfigureAwait(false);
if (current is null) return false;
var titleChanged = current.Title != title;
await UpsertRowAsync(session, current with
{
Folder = folder,
Title = title,
SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
UpdatedAt = DateTimeOffset.UtcNow,
}, cancellationToken).ConfigureAwait(false);
return true;
}, cancellationToken);
private static List<string> Filter(SequenceQuery query, SqlParameters p)
{
var where = new List<string>();