merge: archive layout, sidecars, scan/rebuild/verify, card import and ingest
This commit is contained in:
65 files changed
+7197
No files matched your search
@@ -0,0 +1,46 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Source-generated metadata for every JSON file of the archive layout.</summary>
|
||||
[JsonSourceGenerationOptions(
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
PropertyNameCaseInsensitive = true,
|
||||
AllowTrailingCommas = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip)]
|
||||
[JsonSerializable(typeof(ArchiveManifest))]
|
||||
[JsonSerializable(typeof(SequenceManifest))]
|
||||
[JsonSerializable(typeof(LeaseRecord))]
|
||||
[JsonSerializable(typeof(IngestJournal))]
|
||||
[JsonSerializable(typeof(RenameJournal))]
|
||||
internal sealed partial class ArchiveJson : JsonSerializerContext;
|
||||
|
||||
/// <summary>Indented output for the small files people may open (<c>archive.json</c>, rename journals).</summary>
|
||||
[JsonSourceGenerationOptions(
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
||||
[JsonSerializable(typeof(ArchiveManifest))]
|
||||
[JsonSerializable(typeof(RenameJournal))]
|
||||
internal sealed partial class ArchiveJsonIndented : JsonSerializerContext;
|
||||
|
||||
/// <summary>
|
||||
/// Writes enum names and reads unknown names (written by a newer build) or numbers as the default value instead of
|
||||
/// failing, so a sidecar from a newer Titano is still readable.
|
||||
/// </summary>
|
||||
public sealed class TolerantEnumConverter<TEnum> : JsonConverter<TEnum>
|
||||
where TEnum : struct, Enum
|
||||
{
|
||||
public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
if (reader.TokenType == JsonTokenType.String && Enum.TryParse<TEnum>(reader.GetString(), ignoreCase: true, out var value)
|
||||
&& Enum.IsDefined(value))
|
||||
return value;
|
||||
if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) reader.Skip();
|
||||
return default;
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) => writer.WriteStringValue(value.ToString());
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Names of the fixed files and folders of the archive layout (BRIEF §7).</summary>
|
||||
public static class ArchiveLayout
|
||||
{
|
||||
/// <summary>Metadata folder at the archive root and inside every sequence folder.</summary>
|
||||
public const string MetaFolder = ".titano";
|
||||
|
||||
public const string ArchiveManifestFile = "archive.json";
|
||||
public const string IndexSnapshotFile = "index-snapshot.bin";
|
||||
public const string LocksFolder = "locks";
|
||||
|
||||
/// <summary>Verified copies waiting to be committed; on the archive volume so commits are renames.</summary>
|
||||
public const string StagingFolder = "staging";
|
||||
|
||||
/// <summary>Resumable plans of mass operations (rename).</summary>
|
||||
public const string JournalFolder = "journal";
|
||||
|
||||
public const string OriginalsFolder = "originals";
|
||||
public const string ProxiesFolder = "proxies";
|
||||
public const string ExportsFolder = "exports";
|
||||
public const string SequenceManifestFile = "sequence.json";
|
||||
public const string AnalysisFile = "analysis.json";
|
||||
public const string ProjectFile = "project.json";
|
||||
public const string ThumbsPackFile = "thumbs.pack";
|
||||
|
||||
/// <summary>Moves of an ingest batch that were started but not yet recorded in <see cref="SequenceManifestFile"/>.</summary>
|
||||
public const string IngestJournalFile = "ingest.pending.json";
|
||||
|
||||
/// <summary>Folder names inside a sequence folder that never hold a sequence of their own.</summary>
|
||||
public static bool IsReservedFolder(string name) =>
|
||||
name.Equals(MetaFolder, StringComparison.OrdinalIgnoreCase)
|
||||
|| name.Equals(OriginalsFolder, StringComparison.OrdinalIgnoreCase)
|
||||
|| name.Equals(ProxiesFolder, StringComparison.OrdinalIgnoreCase)
|
||||
|| name.Equals(ExportsFolder, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary><c>0001.jpg</c> for frame index 0: proxies are numbered from 1 like the frames of a render.</summary>
|
||||
public static string ProxyFileName(int index) => (index + 1).ToString("0000", CultureInfo.InvariantCulture) + ".jpg";
|
||||
}
|
||||
|
||||
/// <summary>Absolute paths of one archive root.</summary>
|
||||
public sealed class ArchivePaths
|
||||
{
|
||||
public ArchivePaths(string root)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(root);
|
||||
Root = LongPath.Normalize(Path.TrimEndingDirectorySeparator(Path.GetFullPath(root)));
|
||||
MetaDirectory = Path.Combine(Root, ArchiveLayout.MetaFolder);
|
||||
}
|
||||
|
||||
public string Root { get; }
|
||||
public string MetaDirectory { get; }
|
||||
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.ArchiveManifestFile);
|
||||
public string SnapshotPath => Path.Combine(MetaDirectory, ArchiveLayout.IndexSnapshotFile);
|
||||
public string LocksDirectory => Path.Combine(MetaDirectory, ArchiveLayout.LocksFolder);
|
||||
public string StagingDirectory => Path.Combine(MetaDirectory, ArchiveLayout.StagingFolder);
|
||||
public string JournalDirectory => Path.Combine(MetaDirectory, ArchiveLayout.JournalFolder);
|
||||
|
||||
/// <summary>Paths of the sequence stored in <paramref name="relativeFolder"/> (forward slashes).</summary>
|
||||
public SequencePaths Sequence(string relativeFolder) => new(this, relativeFolder);
|
||||
|
||||
/// <summary>Archive-relative path with forward slashes, as stored in the index.</summary>
|
||||
public string ToRelative(string path) => LongPath.ToPortableRelative(Root, path);
|
||||
|
||||
public string FromRelative(string relative) => LongPath.FromPortableRelative(Root, relative);
|
||||
}
|
||||
|
||||
/// <summary>Absolute paths of one sequence folder.</summary>
|
||||
public sealed class SequencePaths
|
||||
{
|
||||
internal SequencePaths(ArchivePaths archive, string relativeFolder)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(relativeFolder);
|
||||
RelativeFolder = relativeFolder.Replace('\\', '/').Trim('/');
|
||||
Folder = archive.FromRelative(RelativeFolder);
|
||||
MetaDirectory = Path.Combine(Folder, ArchiveLayout.MetaFolder);
|
||||
}
|
||||
|
||||
/// <summary>Folder relative to the archive root, forward slashes; empty when the root itself holds the photos (adopted folder).</summary>
|
||||
public string RelativeFolder { get; }
|
||||
|
||||
public string Folder { get; }
|
||||
public string MetaDirectory { get; }
|
||||
public string Originals => Path.Combine(Folder, ArchiveLayout.OriginalsFolder);
|
||||
public string Proxies => Path.Combine(Folder, ArchiveLayout.ProxiesFolder);
|
||||
public string Exports => Path.Combine(Folder, ArchiveLayout.ExportsFolder);
|
||||
public string ManifestPath => Path.Combine(MetaDirectory, ArchiveLayout.SequenceManifestFile);
|
||||
public string AnalysisPath => Path.Combine(MetaDirectory, ArchiveLayout.AnalysisFile);
|
||||
public string ProjectPath => Path.Combine(MetaDirectory, ArchiveLayout.ProjectFile);
|
||||
public string ThumbsPackPath => Path.Combine(MetaDirectory, ArchiveLayout.ThumbsPackFile);
|
||||
public string IngestJournalPath => Path.Combine(MetaDirectory, ArchiveLayout.IngestJournalFile);
|
||||
|
||||
public string Proxy(int index) => Path.Combine(Proxies, ArchiveLayout.ProxyFileName(index));
|
||||
|
||||
/// <summary>Absolute path of a frame file given its sidecar path (relative to the sequence folder).</summary>
|
||||
public string File(string sequenceRelative) => LongPath.FromPortableRelative(Folder, sequenceRelative);
|
||||
|
||||
/// <summary>Archive-relative path of a frame file (as stored in the frames table).</summary>
|
||||
public string ArchiveRelative(string sequenceRelative) =>
|
||||
RelativeFolder.Length == 0 ? sequenceRelative : RelativeFolder + "/" + sequenceRelative;
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// <c>.titano/archive.json</c> at the archive root (BRIEF §7). Fields this build does not know are kept in
|
||||
/// <see cref="Extra"/> and written back unchanged.
|
||||
/// </summary>
|
||||
public sealed record ArchiveManifest
|
||||
{
|
||||
public const int CurrentSchemaVersion = 1;
|
||||
|
||||
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
|
||||
public required Guid Id { get; init; }
|
||||
public required string Name { get; init; }
|
||||
public DateTimeOffset Created { get; init; }
|
||||
public LayoutTemplates LayoutTemplates { get; init; } = new();
|
||||
public ProxyPolicy ProxyPolicy { get; init; } = new();
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Naming templates used when frames enter the archive (BRIEF §7.1).</summary>
|
||||
public sealed record LayoutTemplates
|
||||
{
|
||||
public string Folder { get; init; } = TemplateDefaults.Folder;
|
||||
public string File { get; init; } = TemplateDefaults.File;
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Proxy generation policy (BRIEF §8.5).</summary>
|
||||
public sealed record ProxyPolicy
|
||||
{
|
||||
public const int Disabled = -1;
|
||||
public const int FullSize = 0;
|
||||
|
||||
/// <summary>Long edge in pixels: 1280, 1920 (default) or 2560; <see cref="FullSize"/> or <see cref="Disabled"/>.</summary>
|
||||
public int LongEdge { get; init; } = 1920;
|
||||
|
||||
public int Quality { get; init; } = 92;
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>A folder that holds a sequence: it has a sidecar, or photos to adopt.</summary>
|
||||
/// <param name="RelativeFolder">Archive-relative, forward slashes; empty for the root.</param>
|
||||
/// <param name="Sidecar">Stamp of <c>.titano/sequence.json</c>, null when it has none yet.</param>
|
||||
/// <param name="Photos">Stamp of the photo directory (its time changes when files are added or removed).</param>
|
||||
public sealed record SequenceFolder(string RelativeFolder, FileStamp? Sidecar, FileStamp? Photos);
|
||||
|
||||
/// <summary>
|
||||
/// Walks an archive tree (BRIEF §7). No file-system watcher is involved: callers compare the stamps with those of
|
||||
/// the previous scan. Hidden folders, symbolic links and the reserved sequence subfolders are not entered.
|
||||
/// </summary>
|
||||
public static class ArchiveScanner
|
||||
{
|
||||
private const int MaxDepth = 16;
|
||||
|
||||
private static readonly EnumerationOptions Listing = new()
|
||||
{
|
||||
IgnoreInaccessible = true,
|
||||
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System,
|
||||
RecurseSubdirectories = false,
|
||||
};
|
||||
|
||||
/// <summary>Every sequence folder under the root, ordered by relative path (ordinal) so job checkpoints are stable.</summary>
|
||||
public static IReadOnlyList<SequenceFolder> FindSequenceFolders(ArchivePaths archive, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var found = new List<SequenceFolder>();
|
||||
var pending = new Stack<(string Path, int Depth)>();
|
||||
pending.Push((archive.Root, 0));
|
||||
while (pending.Count > 0)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var (path, depth) = pending.Pop();
|
||||
var relative = archive.ToRelative(path);
|
||||
var sequence = archive.Sequence(relative == "." ? string.Empty : relative);
|
||||
var sidecar = FileStamp.Of(sequence.ManifestPath);
|
||||
var photoDirectory = PhotoDirectory(sequence);
|
||||
if (sidecar is not null || EnumeratePhotos(photoDirectory).Any())
|
||||
found.Add(new SequenceFolder(sequence.RelativeFolder, sidecar, FileStamp.Of(photoDirectory)));
|
||||
if (depth >= MaxDepth) continue;
|
||||
foreach (var child in Directory.EnumerateDirectories(path, "*", Listing))
|
||||
{
|
||||
var name = Path.GetFileName(child);
|
||||
if (!IsHidden(child, name) && !ArchiveLayout.IsReservedFolder(name)) pending.Push((child, depth + 1));
|
||||
}
|
||||
}
|
||||
found.Sort((a, b) => string.CompareOrdinal(a.RelativeFolder, b.RelativeFolder));
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary><c>originals/</c> when the folder follows the layout, otherwise the folder itself (adopted in place).</summary>
|
||||
public static string PhotoDirectory(SequencePaths sequence) => Directory.Exists(sequence.Originals) ? sequence.Originals : sequence.Folder;
|
||||
|
||||
/// <summary>Photos of a sequence folder as sidecar paths (relative to the sequence folder), ordinal order.</summary>
|
||||
public static IReadOnlyList<string> ListPhotos(SequencePaths sequence)
|
||||
{
|
||||
var directory = PhotoDirectory(sequence);
|
||||
return EnumeratePhotos(directory)
|
||||
.Select(path => Path.GetRelativePath(sequence.Folder, path).Replace('\\', '/'))
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>True for files Titano imports as photos (AppleDouble <c>._</c> companions excluded).</summary>
|
||||
public static bool IsPhoto(string fileName) =>
|
||||
!fileName.StartsWith("._", StringComparison.Ordinal) && ImageFormatInfo.IsSupportedExtension(fileName);
|
||||
|
||||
private static IEnumerable<string> EnumeratePhotos(string directory) =>
|
||||
Directory.Exists(directory)
|
||||
? Directory.EnumerateFiles(directory, "*", Listing).Where(f => IsPhoto(Path.GetFileName(f)) && !IsHidden(f, Path.GetFileName(f)))
|
||||
: [];
|
||||
|
||||
private static bool IsHidden(string path, string name) =>
|
||||
name.StartsWith('.') || (OperatingSystem.IsWindows() && File.GetAttributes(path).HasFlag(FileAttributes.Hidden));
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>The archive cannot be reached (or written) right now; nothing was changed.</summary>
|
||||
public sealed class ArchiveOfflineException(string root, ArchiveAvailability availability)
|
||||
: IOException(Loc.L(availability == ArchiveAvailability.ReadOnly ? "Archive.Error.ReadOnly" : "Archive.Error.Offline", root))
|
||||
{
|
||||
public string Root { get; } = root;
|
||||
public ArchiveAvailability Availability { get; } = availability;
|
||||
}
|
||||
|
||||
public enum ArchiveAvailability
|
||||
{
|
||||
Online,
|
||||
|
||||
/// <summary>Readable but not writable (read-only share, write-protected disk): browse only.</summary>
|
||||
ReadOnly,
|
||||
|
||||
/// <summary>Root unreachable (unplugged drive, disconnected share) or no longer an archive.</summary>
|
||||
Offline,
|
||||
}
|
||||
|
||||
/// <summary>Offline detection for archives on removable and network volumes (BRIEF §7).</summary>
|
||||
public static class ArchiveVolume
|
||||
{
|
||||
/// <summary>Throws <see cref="ArchiveOfflineException"/> unless the archive is online (writable).</summary>
|
||||
public static void EnsureWritable(ArchivePaths archive)
|
||||
{
|
||||
var availability = Probe(archive);
|
||||
if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(archive.Root, availability);
|
||||
}
|
||||
|
||||
/// <summary>Checks that <c>archive.json</c> is reachable and that the metadata folder accepts writes.</summary>
|
||||
public static ArchiveAvailability Probe(ArchivePaths archive)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(archive.ManifestPath)) return ArchiveAvailability.Offline;
|
||||
var probe = Path.Combine(archive.MetaDirectory, ".probe-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1, FileOptions.DeleteOnClose))
|
||||
{
|
||||
}
|
||||
return ArchiveAvailability.Online;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return ArchiveAvailability.ReadOnly;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return File.Exists(archive.ManifestPath) ? ArchiveAvailability.ReadOnly : ArchiveAvailability.Offline;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>SHA-256 of files, streamed (originals can be hundreds of megabytes).</summary>
|
||||
public static class ContentHash
|
||||
{
|
||||
internal const int BufferSize = 1 << 20;
|
||||
|
||||
/// <summary>Lower-case hex SHA-256 of the file.</summary>
|
||||
/// <param name="progress">Receives the number of bytes read since the previous call.</param>
|
||||
public static async Task<string> Sha256Async(string path, Action<long>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, BufferSize,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
return await Sha256Async(stream, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public static async Task<string> Sha256Async(Stream stream, Action<long>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
var buffer = new byte[BufferSize];
|
||||
int read;
|
||||
while ((read = await stream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
progress?.Invoke(read);
|
||||
}
|
||||
return Convert.ToHexStringLower(hash.GetHashAndReset());
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Size and modification time of a file, the cheap change detector of incremental scans (BRIEF §7).</summary>
|
||||
public readonly record struct FileStamp(long Size, DateTime ModifiedUtc)
|
||||
{
|
||||
/// <summary>The stamp of a file or directory, or null when it does not exist.</summary>
|
||||
public static FileStamp? Of(string path)
|
||||
{
|
||||
var file = new FileInfo(path);
|
||||
if (file.Exists) return new FileStamp(file.Length, file.LastWriteTimeUtc);
|
||||
var directory = new DirectoryInfo(path);
|
||||
return directory.Exists ? new FileStamp(0, directory.LastWriteTimeUtc) : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using Titano.Core.Codecs.Jpeg;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Metadata;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>A JPEG made from a frame.</summary>
|
||||
public sealed record EncodedImage(byte[] Jpeg, int Width, int Height, DecodeTier Tier);
|
||||
|
||||
/// <summary>
|
||||
/// Makes thumbnails and proxies of frames (BRIEF §8.5): the embedded preview or the managed JPEG decoder first, then
|
||||
/// the host's decode providers (WIC on Windows); orientation applied, resampled in linear light, encoded in-house.
|
||||
/// </summary>
|
||||
public sealed class FrameImageRenderer(ImageDecoder decoder)
|
||||
{
|
||||
public const int ThumbnailLongEdge = 320;
|
||||
public const int ThumbnailQuality = 80;
|
||||
|
||||
/// <summary>A 320 px q80 thumbnail, or null when no decoder can read the frame.</summary>
|
||||
public EncodedImage? Thumbnail(SequencePaths sequence, SidecarFrame frame, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var decoded = Decode(sequence.File(frame.File), frame, ThumbnailLongEdge, DecodePurpose.Thumbnail, preferPreview: true, cancellationToken);
|
||||
if (decoded is null) return null;
|
||||
using var small = Resampler.Thumbnail(decoded.Pixels, ThumbnailLongEdge);
|
||||
return Encode(small, new JpegEncodeOptions { Quality = ThumbnailQuality }, decoded.Tier);
|
||||
}
|
||||
|
||||
/// <summary>A proxy at the policy's long edge and quality with capture time and camera in its Exif; null when undecodable.</summary>
|
||||
public EncodedImage? Proxy(SequencePaths sequence, SidecarFrame frame, SidecarCamera? camera, ProxyPolicy policy, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (policy.LongEdge == ProxyPolicy.Disabled) throw new InvalidOperationException("Proxies are disabled for this archive.");
|
||||
using var decoded = Decode(sequence.File(frame.File), frame, policy.LongEdge, DecodePurpose.Proxy, preferPreview: false, cancellationToken);
|
||||
if (decoded is null) return null;
|
||||
var source = decoded.Pixels;
|
||||
var (w, h) = policy.LongEdge == ProxyPolicy.FullSize ? (source.Width, source.Height) : Resampler.FitWithin(source.Width, source.Height, policy.LongEdge);
|
||||
using var resized = Resampler.Resize(source, w, h);
|
||||
var exif = ExifWriter.BuildJpegApp1(new FrameMetadata
|
||||
{
|
||||
Make = camera?.Make,
|
||||
Model = camera?.Model,
|
||||
BodySerial = camera?.Serial,
|
||||
CapturedAt = frame.CapturedAt,
|
||||
UtcOffsetMinutes = frame.UtcOffset,
|
||||
ExposureTime = frame.Exposure,
|
||||
FNumber = frame.FNumber,
|
||||
Iso = frame.Iso,
|
||||
FocalLength = frame.Focal,
|
||||
ExposureBias = frame.Bias,
|
||||
Software = "Titano",
|
||||
});
|
||||
// The encoder takes the APP1 payload; ExifWriter returns the whole segment (marker and length first).
|
||||
return Encode(resized, new JpegEncodeOptions { Quality = policy.Quality, App1Segments = [exif[4..]] }, decoded.Tier);
|
||||
}
|
||||
|
||||
private static EncodedImage Encode(ImageBuffer pixels, JpegEncodeOptions options, DecodeTier tier) =>
|
||||
new(JpegEncoder.Encode(pixels, options), pixels.Width, pixels.Height, tier);
|
||||
|
||||
private DecodedFrame? Decode(string path, SidecarFrame frame, int maxLongEdge, DecodePurpose purpose, bool preferPreview, CancellationToken cancellationToken)
|
||||
{
|
||||
if (frame.Format == ImageFileFormat.Jpeg) return DecodeJpeg(File.ReadAllBytes(path), maxLongEdge, frame.Orientation, DecodeTier.Native);
|
||||
if (preferPreview && ReadPreview(path, frame) is { } preview) return DecodeJpeg(preview, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview);
|
||||
var request = new DecodeRequest(path, System.Math.Max(0, maxLongEdge), purpose, ApplyOrientation: true, AllowPreview: true);
|
||||
if (decoder.Decode(request, cancellationToken) is { } decoded) return new DecodedFrame(decoded.Pixels, decoded.Tier);
|
||||
return ReadPreview(path, frame) is { } fallback ? DecodeJpeg(fallback, maxLongEdge, frame.Orientation, DecodeTier.EmbeddedPreview) : null;
|
||||
}
|
||||
|
||||
private static DecodedFrame? DecodeJpeg(byte[] data, int maxLongEdge, int orientation, DecodeTier tier)
|
||||
{
|
||||
ImageBuffer decoded;
|
||||
try
|
||||
{
|
||||
decoded = JpegDecoder.DecodeToLinear(data, new JpegDecodeOptions { MaxLongEdge = System.Math.Max(0, maxLongEdge) }, expandGray: true);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
if (orientation is <= 1 or > 8) return new DecodedFrame(decoded, tier);
|
||||
using (decoded) return new DecodedFrame(PixelConvert.ApplyOrientation(decoded, orientation), tier);
|
||||
}
|
||||
|
||||
private static byte[]? ReadPreview(string path, SidecarFrame frame)
|
||||
{
|
||||
if (frame.PreviewOffset is not { } offset || frame.PreviewLength is not ({ } length and > 0 and < int.MaxValue)) return null;
|
||||
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete, 4096, FileOptions.RandomAccess);
|
||||
if (offset < 0 || offset + length > stream.Length) return null;
|
||||
var data = new byte[length];
|
||||
stream.Position = offset;
|
||||
stream.ReadExactly(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private sealed record DecodedFrame(ImageBuffer Pixels, DecodeTier Tier) : IDisposable
|
||||
{
|
||||
public void Dispose() => Pixels.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
public enum FrameHealth
|
||||
{
|
||||
/// <summary>Same size and SHA-256 as recorded.</summary>
|
||||
Ok,
|
||||
|
||||
/// <summary>No hash was recorded (adopted file); it has been computed now.</summary>
|
||||
Hashed,
|
||||
|
||||
Missing,
|
||||
|
||||
/// <summary>A different size: the file was replaced or edited.</summary>
|
||||
Changed,
|
||||
|
||||
/// <summary>Same size, different content: bit rot or a damaged copy.</summary>
|
||||
Corrupt,
|
||||
}
|
||||
|
||||
/// <param name="Sha256">The hash computed now (null when the file is missing or changed size).</param>
|
||||
public sealed record FrameCheck(SidecarFrame Frame, FrameHealth Health, string? Sha256)
|
||||
{
|
||||
/// <summary>Frame flags after this check: <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> set or cleared.</summary>
|
||||
public FrameFlags Flags => Health switch
|
||||
{
|
||||
FrameHealth.Missing => (Frame.Flags & ~FrameFlags.Corrupt) | FrameFlags.Missing,
|
||||
FrameHealth.Changed or FrameHealth.Corrupt => (Frame.Flags & ~FrameFlags.Missing) | FrameFlags.Corrupt,
|
||||
_ => Frame.Flags & ~(FrameFlags.Missing | FrameFlags.Corrupt),
|
||||
};
|
||||
|
||||
/// <summary>The sidecar record after this check (flags, and the hash of a newly hashed file).</summary>
|
||||
public SidecarFrame Updated => Frame with { Flags = Flags, Sha256 = Health == FrameHealth.Hashed ? Sha256 : Frame.Sha256 };
|
||||
|
||||
public bool ChangesRecord => Updated != Frame;
|
||||
}
|
||||
|
||||
/// <summary>Re-hashes originals against their sidecar records (BRIEF §12.5 Verify).</summary>
|
||||
public static class FrameVerifier
|
||||
{
|
||||
public static async Task<FrameCheck> CheckAsync(SequencePaths sequence, SidecarFrame frame, Action<long>? bytesRead = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var info = new FileInfo(sequence.File(frame.File));
|
||||
if (!info.Exists) return new FrameCheck(frame, FrameHealth.Missing, null);
|
||||
if (info.Length != frame.Size) return new FrameCheck(frame, FrameHealth.Changed, null);
|
||||
var sha = await ContentHash.Sha256Async(info.FullName, bytesRead, cancellationToken).ConfigureAwait(false);
|
||||
var health = frame.Sha256 is null ? FrameHealth.Hashed
|
||||
: sha.Equals(frame.Sha256, StringComparison.OrdinalIgnoreCase) ? FrameHealth.Ok
|
||||
: FrameHealth.Corrupt;
|
||||
return new FrameCheck(frame, health, sha);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// <c>.titano/index-snapshot.bin</c> (BRIEF §7): every sidecar of the archive in one file, so a rebuild reads one file
|
||||
/// sequentially instead of thousands of small ones over a share. Optional and never authoritative: an entry is used
|
||||
/// only when its sidecar still has the recorded size and time, and a file whose trailing SHA-256 does not match is
|
||||
/// ignored entirely.
|
||||
/// </summary>
|
||||
/// <remarks>Layout: "TISN", version (u16), reserved (u16), then per sidecar: folder length (u16), UTF-8 folder,
|
||||
/// size (i64), modified UTC ticks (i64), length (i32), sidecar bytes; finally the SHA-256 of everything before it.</remarks>
|
||||
public sealed class IndexSnapshot
|
||||
{
|
||||
private const ushort Version = 1;
|
||||
private static ReadOnlySpan<byte> Magic => "TISN"u8;
|
||||
|
||||
private readonly Dictionary<string, (FileStamp Stamp, byte[] Bytes)> _entries;
|
||||
|
||||
private IndexSnapshot(Dictionary<string, (FileStamp, byte[])> entries) => _entries = entries;
|
||||
|
||||
public int Count => _entries.Count;
|
||||
|
||||
/// <summary>The snapshot at <paramref name="path"/>, or null when it is absent, damaged or of another version.</summary>
|
||||
public static IndexSnapshot? TryLoad(string path)
|
||||
{
|
||||
byte[] data;
|
||||
try
|
||||
{
|
||||
if (!File.Exists(path)) return null;
|
||||
data = File.ReadAllBytes(path);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
if (data.Length < 8 + 32 || !data.AsSpan(0, 4).SequenceEqual(Magic) || BinaryPrimitives.ReadUInt16LittleEndian(data.AsSpan(4)) != Version)
|
||||
return null;
|
||||
var body = data.AsSpan(0, data.Length - 32);
|
||||
if (!SHA256.HashData(body).AsSpan().SequenceEqual(data.AsSpan(data.Length - 32))) return null;
|
||||
|
||||
var entries = new Dictionary<string, (FileStamp, byte[])>(StringComparer.Ordinal);
|
||||
var position = 8;
|
||||
while (position < body.Length)
|
||||
{
|
||||
if (position + 2 > body.Length) return null;
|
||||
var folderLength = BinaryPrimitives.ReadUInt16LittleEndian(body[position..]);
|
||||
position += 2;
|
||||
if (position + folderLength + 20 > body.Length) return null;
|
||||
var folder = Encoding.UTF8.GetString(body.Slice(position, folderLength));
|
||||
position += folderLength;
|
||||
var size = BinaryPrimitives.ReadInt64LittleEndian(body[position..]);
|
||||
var ticks = BinaryPrimitives.ReadInt64LittleEndian(body[(position + 8)..]);
|
||||
var length = BinaryPrimitives.ReadInt32LittleEndian(body[(position + 16)..]);
|
||||
position += 20;
|
||||
if (length < 0 || ticks < 0 || ticks > DateTime.MaxValue.Ticks || position + length > body.Length) return null;
|
||||
entries[folder] = (new FileStamp(size, new DateTime(ticks, DateTimeKind.Utc)), body.Slice(position, length).ToArray());
|
||||
position += length;
|
||||
}
|
||||
return new IndexSnapshot(entries);
|
||||
}
|
||||
|
||||
/// <summary>The recorded sidecar bytes of <paramref name="folder"/> when its current stamp still matches.</summary>
|
||||
public byte[]? TryGet(string folder, FileStamp current) =>
|
||||
_entries.TryGetValue(folder, out var entry) && entry.Stamp == current ? entry.Bytes : null;
|
||||
|
||||
/// <summary>Streams a new snapshot to a temporary file; <see cref="Commit"/> replaces the old one atomically.</summary>
|
||||
public sealed class Writer : IDisposable
|
||||
{
|
||||
private readonly string _path;
|
||||
private readonly string _temp;
|
||||
private readonly FileStream _stream;
|
||||
private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
private bool _committed;
|
||||
|
||||
public Writer(string path)
|
||||
{
|
||||
_path = path;
|
||||
_temp = AtomicFile.TempPathFor(path);
|
||||
_stream = new FileStream(_temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16);
|
||||
Span<byte> header = stackalloc byte[8];
|
||||
header.Clear();
|
||||
Magic.CopyTo(header);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
|
||||
Write(header);
|
||||
}
|
||||
|
||||
public void Add(string folder, FileStamp stamp, ReadOnlySpan<byte> sidecar)
|
||||
{
|
||||
var name = Encoding.UTF8.GetBytes(folder);
|
||||
Span<byte> head = stackalloc byte[2];
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(head, checked((ushort)name.Length));
|
||||
Write(head);
|
||||
Write(name);
|
||||
Span<byte> fields = stackalloc byte[20];
|
||||
BinaryPrimitives.WriteInt64LittleEndian(fields, stamp.Size);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(fields[8..], stamp.ModifiedUtc.Ticks);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(fields[16..], sidecar.Length);
|
||||
Write(fields);
|
||||
Write(sidecar);
|
||||
}
|
||||
|
||||
public void Commit()
|
||||
{
|
||||
_stream.Write(_hash.GetHashAndReset());
|
||||
_stream.Dispose();
|
||||
AtomicFile.Commit(_temp, _path);
|
||||
_committed = true;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_stream.Dispose();
|
||||
_hash.Dispose();
|
||||
if (!_committed && File.Exists(_temp)) File.Delete(_temp);
|
||||
}
|
||||
|
||||
private void Write(ReadOnlySpan<byte> data)
|
||||
{
|
||||
_hash.AppendData(data);
|
||||
_stream.Write(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Write-ahead record of an ingest batch (<c>.titano/ingest.pending.json</c>): the frames whose files are about to be
|
||||
/// moved into the sequence folder. Written before the moves and removed once <c>sequence.json</c> lists them, so a
|
||||
/// crash in between is repaired by <see cref="Recover"/> instead of leaving unindexed originals behind.
|
||||
/// </summary>
|
||||
public sealed record IngestJournal(IReadOnlyList<SidecarFrame> Frames)
|
||||
{
|
||||
public static void Write(SequencePaths sequence, IngestJournal journal)
|
||||
{
|
||||
Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
AtomicFile.WriteAllBytes(sequence.IngestJournalPath, JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJson.Default.IngestJournal));
|
||||
}
|
||||
|
||||
public static void Delete(SequencePaths sequence)
|
||||
{
|
||||
if (File.Exists(sequence.IngestJournalPath)) File.Delete(sequence.IngestJournalPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Completes an interrupted batch: frames whose file reached the sequence folder (same size) but are missing from
|
||||
/// <paramref name="manifest"/> are added in capture order. Returns the manifest to write, or null when nothing has
|
||||
/// to change. The journal is left in place; <see cref="Delete"/> it once the result is written.
|
||||
/// </summary>
|
||||
public static SequenceManifest? Recover(SequencePaths sequence, SequenceManifest manifest)
|
||||
{
|
||||
if (!File.Exists(sequence.IngestJournalPath)) return null;
|
||||
IngestJournal? journal;
|
||||
try
|
||||
{
|
||||
journal = JsonSerializer.Deserialize(File.ReadAllBytes(sequence.IngestJournalPath), ArchiveJson.Default.IngestJournal);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
journal = null; // torn while being written: nothing was moved yet
|
||||
}
|
||||
if (journal is null) return null;
|
||||
var known = manifest.Frames.Select(f => f.Id).ToHashSet();
|
||||
var moved = journal.Frames.Where(f => !known.Contains(f.Id) && FileHasSize(sequence.File(f.File), f.Size)).ToList();
|
||||
return moved.Count == 0 ? null : SequenceFrames.Insert(manifest, moved);
|
||||
}
|
||||
|
||||
private static bool FileHasSize(string path, long size)
|
||||
{
|
||||
var info = new FileInfo(path);
|
||||
return info.Exists && info.Length == size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Content of a lease file: who holds it and until when.</summary>
|
||||
public sealed record LeaseRecord(string Token, string Owner, string Host, int ProcessId, DateTimeOffset Acquired, DateTimeOffset Expires);
|
||||
|
||||
/// <summary>The lease is held by someone else and did not free up in time.</summary>
|
||||
public sealed class LeaseUnavailableException(string name, LeaseRecord? holder)
|
||||
: IOException(Loc.L("Archive.Error.Busy", holder?.Owner ?? "?"))
|
||||
{
|
||||
public string Name { get; } = name;
|
||||
public LeaseRecord? Holder { get; } = holder;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cross-process exclusive lease stored as <c>.titano/locks/<name>.lease</c> (BRIEF §7), for writers that share
|
||||
/// an archive over a network (server and client). Creation uses create-new semantics, which SMB and local file
|
||||
/// systems both make atomic. A lease past its expiry is stale (its holder crashed or lost the share) and is taken
|
||||
/// over by renaming it aside, which only one contender can do. The holder renews it in the background while held.
|
||||
/// </summary>
|
||||
public sealed class LeaseFile : IDisposable
|
||||
{
|
||||
public static readonly TimeSpan DefaultDuration = TimeSpan.FromMinutes(2);
|
||||
|
||||
private const string Extension = ".lease";
|
||||
private const string StaleExtension = ".stale";
|
||||
private readonly Lock _gate = new();
|
||||
private readonly Timer _renewal;
|
||||
private LeaseRecord _record;
|
||||
private bool _released;
|
||||
|
||||
private LeaseFile(string path, string name, LeaseRecord record, TimeSpan duration)
|
||||
{
|
||||
Path = path;
|
||||
Name = name;
|
||||
_record = record;
|
||||
Duration = duration;
|
||||
var period = duration / 3;
|
||||
_renewal = new Timer(_ => Renew(), null, period, period);
|
||||
}
|
||||
|
||||
public string Path { get; }
|
||||
public string Name { get; }
|
||||
public TimeSpan Duration { get; }
|
||||
public LeaseRecord Record => _record;
|
||||
|
||||
/// <summary>False once a renewal found the lease taken over (it expired while this process was stalled).</summary>
|
||||
public bool IsHeld { get; private set; } = true;
|
||||
|
||||
/// <summary>Takes the lease when it is free or stale; null when someone else holds it.</summary>
|
||||
public static LeaseFile? TryAcquire(string locksDirectory, string name, string owner, TimeSpan? duration = null)
|
||||
{
|
||||
var length = duration ?? DefaultDuration;
|
||||
var path = PathFor(locksDirectory, name);
|
||||
Directory.CreateDirectory(locksDirectory);
|
||||
for (var attempt = 0; attempt < 3; attempt++)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var record = new LeaseRecord(Guid.NewGuid().ToString("N"), owner, Environment.MachineName, Environment.ProcessId, now, now + length);
|
||||
if (TryCreate(path, record)) return new LeaseFile(path, name, record, length);
|
||||
if (!IsStale(path, now)) return null;
|
||||
BreakStale(path, now);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>Waits up to <paramref name="wait"/> for the lease, polling with back-off.</summary>
|
||||
/// <exception cref="LeaseUnavailableException">Still held by someone else after <paramref name="wait"/>.</exception>
|
||||
public static async Task<LeaseFile> AcquireAsync(string locksDirectory, string name, string owner, TimeSpan wait,
|
||||
TimeSpan? duration = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + wait;
|
||||
var delay = TimeSpan.FromMilliseconds(20);
|
||||
while (true)
|
||||
{
|
||||
if (TryAcquire(locksDirectory, name, owner, duration) is { } lease) return lease;
|
||||
if (DateTime.UtcNow >= deadline) throw new LeaseUnavailableException(name, Read(PathFor(locksDirectory, name)));
|
||||
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
|
||||
delay = TimeSpan.FromMilliseconds(System.Math.Min(delay.TotalMilliseconds * 2, 500));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The current holder of a lease, or null when free or unreadable.</summary>
|
||||
public static LeaseRecord? Read(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.LeaseRecord);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Removes every stale lease in <paramref name="locksDirectory"/> (maintenance); returns their names.</summary>
|
||||
public static IReadOnlyList<string> CleanupStale(string locksDirectory, bool dryRun)
|
||||
{
|
||||
if (!Directory.Exists(locksDirectory)) return [];
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var removed = new List<string>();
|
||||
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + Extension))
|
||||
{
|
||||
if (!IsStale(path, now)) continue;
|
||||
removed.Add(System.IO.Path.GetFileNameWithoutExtension(path));
|
||||
if (!dryRun) BreakStale(path, now);
|
||||
}
|
||||
if (!dryRun)
|
||||
{
|
||||
foreach (var path in Directory.EnumerateFiles(locksDirectory, "*" + StaleExtension))
|
||||
{
|
||||
if (File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime) TryDelete(path);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>Extends the expiry; false when the lease is no longer ours.</summary>
|
||||
public bool Renew()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_released || !IsHeld) return false;
|
||||
if (Read(Path)?.Token != _record.Token)
|
||||
{
|
||||
IsHeld = false;
|
||||
Log.Warning("archive", $"Lease {Name} was taken over by another writer");
|
||||
return false;
|
||||
}
|
||||
var renewed = _record with { Expires = DateTimeOffset.UtcNow + Duration };
|
||||
try
|
||||
{
|
||||
// In-place rewrite: a rename would open a window in which the lease looks free.
|
||||
using var fs = new FileStream(Path, FileMode.Truncate, FileAccess.Write, FileShare.Read);
|
||||
fs.Write(JsonSerializer.SerializeToUtf8Bytes(renewed, ArchiveJson.Default.LeaseRecord));
|
||||
fs.Flush(flushToDisk: true);
|
||||
_record = renewed;
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Runs on a timer thread: an escaping exception would end the process.
|
||||
Log.Warning("archive", $"Lease {Name} could not be renewed", ex);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_renewal.Dispose();
|
||||
lock (_gate)
|
||||
{
|
||||
if (_released) return;
|
||||
_released = true;
|
||||
if (!IsHeld || Read(Path)?.Token != _record.Token) return;
|
||||
try
|
||||
{
|
||||
File.Delete(Path);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// It expires on its own.
|
||||
Log.Warning("archive", $"Lease {Name} could not be released", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal static string PathFor(string locksDirectory, string name) => System.IO.Path.Combine(locksDirectory, name + Extension);
|
||||
|
||||
private static bool TryCreate(string path, LeaseRecord record)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var fs = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.Read);
|
||||
fs.Write(JsonSerializer.SerializeToUtf8Bytes(record, ArchiveJson.Default.LeaseRecord));
|
||||
fs.Flush(flushToDisk: true);
|
||||
return true;
|
||||
}
|
||||
catch (IOException) when (File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Expired, or unreadable for longer than a lease lasts (a holder that crashed while creating it).</summary>
|
||||
private static bool IsStale(string path, DateTimeOffset now)
|
||||
{
|
||||
if (Read(path) is { } record) return record.Expires < now;
|
||||
try
|
||||
{
|
||||
return File.Exists(path) && File.GetLastWriteTimeUtc(path) + DefaultDuration < now.UtcDateTime;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static void BreakStale(string path, DateTimeOffset now)
|
||||
{
|
||||
var aside = path + "." + Guid.NewGuid().ToString("N")[..8] + StaleExtension;
|
||||
try
|
||||
{
|
||||
File.Move(path, aside);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return; // another contender broke it first
|
||||
}
|
||||
if (IsStale(aside, now))
|
||||
{
|
||||
TryDelete(aside);
|
||||
return;
|
||||
}
|
||||
// A contender renewed or re-created the lease between our check and the rename: give it back.
|
||||
try
|
||||
{
|
||||
File.Move(aside, path, overwrite: false);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
TryDelete(aside);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// A leftover file is harmless; maintenance removes it.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>A sidecar written by a newer Titano: it can be read, but this build refuses to rewrite it.</summary>
|
||||
public sealed class NewerFormatException(string path, int version)
|
||||
: InvalidOperationException(Loc.L("Archive.Error.NewerFormat", System.IO.Path.GetFileName(path), version))
|
||||
{
|
||||
public string Path { get; } = path;
|
||||
public int Version { get; } = version;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads and writes the JSON files of the layout. Reading tolerates unknown fields and enum values; every write
|
||||
/// goes through <see cref="AtomicFile"/> (temp → fsync → rename), so a crash leaves the previous file intact.
|
||||
/// </summary>
|
||||
public static class ManifestStore
|
||||
{
|
||||
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
|
||||
public static ArchiveManifest ReadArchive(string path) => Parse(File.ReadAllBytes(path), path, ArchiveJson.Default.ArchiveManifest);
|
||||
|
||||
public static void WriteArchive(string path, ArchiveManifest manifest)
|
||||
{
|
||||
RefuseNewer(path, manifest.SchemaVersion, ArchiveManifest.CurrentSchemaVersion);
|
||||
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJsonIndented.Default.ArchiveManifest));
|
||||
}
|
||||
|
||||
/// <exception cref="InvalidDataException">The file is not valid JSON or lacks required fields.</exception>
|
||||
public static SequenceManifest ReadSequence(string path) => ParseSequence(File.ReadAllBytes(path), path);
|
||||
|
||||
/// <summary>Parses sidecar bytes (from disk or from the index snapshot).</summary>
|
||||
public static SequenceManifest ParseSequence(ReadOnlySpan<byte> utf8, string source) => Parse(utf8, source, ArchiveJson.Default.SequenceManifest);
|
||||
|
||||
/// <summary>Replaces the sidecar atomically. Compact JSON: a 100k-frame sidecar stays around 35 MB.</summary>
|
||||
public static void WriteSequence(string path, SequenceManifest manifest)
|
||||
{
|
||||
RefuseNewer(path, manifest.SchemaVersion, SequenceManifest.CurrentSchemaVersion);
|
||||
AtomicFile.WriteAllBytes(path, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the sidecar only if none exists (rename without replace), which makes it the claim on a new
|
||||
/// sequence folder between processes sharing the archive. False when another sidecar is already there.
|
||||
/// </summary>
|
||||
public static bool TryCreateSequence(string path, SequenceManifest manifest)
|
||||
{
|
||||
if (File.Exists(path)) return false;
|
||||
var temp = AtomicFile.TempPathFor(path);
|
||||
File.WriteAllBytes(temp, JsonSerializer.SerializeToUtf8Bytes(manifest, ArchiveJson.Default.SequenceManifest));
|
||||
try
|
||||
{
|
||||
using (var fs = new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) fs.Flush(flushToDisk: true);
|
||||
File.Move(temp, path, overwrite: false);
|
||||
return true;
|
||||
}
|
||||
catch (IOException) when (File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temp)) File.Delete(temp);
|
||||
}
|
||||
}
|
||||
|
||||
private static T Parse<T>(ReadOnlySpan<byte> utf8, string source, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo)
|
||||
{
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize(utf8, typeInfo) ?? throw new InvalidDataException($"{source} is empty.");
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new InvalidDataException($"{source} is not a valid Titano file: {ex.Message}", ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static void RefuseNewer(string path, int version, int supported)
|
||||
{
|
||||
if (version > supported) throw new NewerFormatException(path, version);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>What the planner needs to know about a sequence to rename it.</summary>
|
||||
public sealed record RenameSubject(Guid SequenceId, string Folder, string? Title, DateTime? FirstCapture, string? CameraMake, string? CameraModel, string? Lens);
|
||||
|
||||
/// <summary>One sequence's change: its folder, its title, or both.</summary>
|
||||
public sealed record RenameStep(Guid SequenceId, string FromFolder, string ToFolder, string? FromTitle, string? ToTitle)
|
||||
{
|
||||
public bool MovesFolder => !string.Equals(FromFolder, ToFolder, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>The exact changes a mass rename will make (its dry run); applying it makes exactly these.</summary>
|
||||
public sealed record RenamePlan(Guid Id, Guid ArchiveId, IReadOnlyList<RenameStep> Steps);
|
||||
|
||||
/// <summary>A plan being applied, with the steps already completed; lets an interrupted rename resume (BRIEF §7).</summary>
|
||||
public sealed record RenameJournal(RenamePlan Plan, IReadOnlyList<Guid> Completed)
|
||||
{
|
||||
public static string PathFor(ArchivePaths archive, Guid planId) => Path.Combine(archive.JournalDirectory, $"rename-{planId:N}.json");
|
||||
|
||||
public static void Write(ArchivePaths archive, RenameJournal journal)
|
||||
{
|
||||
Directory.CreateDirectory(archive.JournalDirectory);
|
||||
AtomicFile.WriteAllBytes(PathFor(archive, journal.Plan.Id), JsonSerializer.SerializeToUtf8Bytes(journal, ArchiveJsonIndented.Default.RenameJournal));
|
||||
}
|
||||
|
||||
/// <summary>Journals left by interrupted renames.</summary>
|
||||
public static IReadOnlyList<RenameJournal> ListPending(ArchivePaths archive)
|
||||
{
|
||||
if (!Directory.Exists(archive.JournalDirectory)) return [];
|
||||
var list = new List<RenameJournal>();
|
||||
foreach (var path in Directory.EnumerateFiles(archive.JournalDirectory, "rename-*.json").Order(StringComparer.Ordinal))
|
||||
{
|
||||
if (JsonSerializer.Deserialize(File.ReadAllBytes(path), ArchiveJson.Default.RenameJournal) is { } journal) list.Add(journal);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static void Delete(ArchivePaths archive, Guid planId) => File.Delete(PathFor(archive, planId));
|
||||
}
|
||||
|
||||
/// <summary>Builds rename plans from folder/title templates (BRIEF §7.1, §12.5).</summary>
|
||||
public static class RenamePlanner
|
||||
{
|
||||
/// <summary>
|
||||
/// Expands <paramref name="titleTemplate"/> (where <c>{title}</c> is the current title) and then
|
||||
/// <paramref name="folderTemplate"/> (where <c>{title}</c> is the new title) for every subject. A target folder
|
||||
/// that is any current sequence folder, exists on disk or was given to an earlier subject gets " (2)", " (3)"….
|
||||
/// Null templates leave that part unchanged; subjects without a change produce no step.
|
||||
/// </summary>
|
||||
/// <param name="occupied">Every sequence folder of the archive (renamed or not).</param>
|
||||
public static RenamePlan Plan(Guid archiveId, IReadOnlyList<RenameSubject> subjects, string? folderTemplate, string? titleTemplate,
|
||||
IEnumerable<string> occupied, Func<string, bool> existsOnDisk)
|
||||
{
|
||||
var folder = folderTemplate is null ? null : PathTemplate.Parse(folderTemplate);
|
||||
var title = titleTemplate is null ? null : PathTemplate.Parse(titleTemplate);
|
||||
var claimed = new HashSet<string>(occupied, StringComparer.OrdinalIgnoreCase);
|
||||
var steps = new List<RenameStep>();
|
||||
foreach (var s in subjects)
|
||||
{
|
||||
var context = new NamingContext
|
||||
{
|
||||
CapturedAt = s.FirstCapture ?? DateTime.Today,
|
||||
SessionStart = s.FirstCapture ?? DateTime.Today,
|
||||
CameraMake = s.CameraMake,
|
||||
CameraModel = s.CameraModel,
|
||||
Lens = s.Lens,
|
||||
Title = s.Title,
|
||||
};
|
||||
var newTitle = title is null ? s.Title : Blank(title.ExpandFile(context)) ?? s.Title;
|
||||
var newFolder = s.Folder;
|
||||
// The archive root itself (a folder adopted in place) cannot move; only its title changes.
|
||||
if (folder is not null && s.Folder.Length > 0 && Blank(folder.ExpandFolder(context with { Title = newTitle })) is { } expanded
|
||||
&& !expanded.Equals(s.Folder, StringComparison.Ordinal))
|
||||
{
|
||||
newFolder = expanded.Equals(s.Folder, StringComparison.OrdinalIgnoreCase) ? expanded : Unique(expanded, claimed, existsOnDisk);
|
||||
claimed.Add(newFolder);
|
||||
}
|
||||
if (newFolder != s.Folder || newTitle != s.Title) steps.Add(new RenameStep(s.SequenceId, s.Folder, newFolder, s.Title, newTitle));
|
||||
}
|
||||
return new RenamePlan(Guid.CreateVersion7(), archiveId, steps);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the folder of a step; idempotent so a resumed plan can repeat it: an already moved folder is accepted,
|
||||
/// a case-only change goes through a temporary name (case-insensitive file systems).
|
||||
/// </summary>
|
||||
/// <exception cref="IOException">The target is occupied by something else, or neither folder exists.</exception>
|
||||
public static void MoveFolder(ArchivePaths archive, RenameStep step)
|
||||
{
|
||||
if (!step.MovesFolder) return;
|
||||
var from = archive.FromRelative(step.FromFolder);
|
||||
var to = archive.FromRelative(step.ToFolder);
|
||||
var caseOnly = step.FromFolder.Equals(step.ToFolder, StringComparison.OrdinalIgnoreCase);
|
||||
if (Directory.Exists(to) && (!caseOnly || HasExactName(to)))
|
||||
{
|
||||
if (Directory.Exists(from) && !caseOnly) throw new IOException($"Rename target {step.ToFolder} already exists.");
|
||||
return;
|
||||
}
|
||||
if (!Directory.Exists(from)) throw new DirectoryNotFoundException($"Sequence folder {step.FromFolder} is missing.");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
|
||||
if (caseOnly)
|
||||
{
|
||||
var temp = from + ".rename-" + Guid.NewGuid().ToString("N")[..8];
|
||||
Directory.Move(from, temp);
|
||||
Directory.Move(temp, to);
|
||||
}
|
||||
else
|
||||
{
|
||||
Directory.Move(from, to);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool HasExactName(string path) =>
|
||||
Directory.EnumerateDirectories(Path.GetDirectoryName(path)!, Path.GetFileName(path))
|
||||
.Any(d => Path.GetFileName(d).Equals(Path.GetFileName(path), StringComparison.Ordinal));
|
||||
|
||||
private static string? Blank(string value) => string.IsNullOrWhiteSpace(value) ? null : value;
|
||||
|
||||
private static string Unique(string folder, HashSet<string> claimed, Func<string, bool> existsOnDisk)
|
||||
{
|
||||
var candidate = folder;
|
||||
for (var n = 2; claimed.Contains(candidate) || existsOnDisk(candidate); n++)
|
||||
candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})");
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Files of a sequence folder that the sidecar does not list, and sidecar frames whose file is gone.</summary>
|
||||
public sealed record SequenceOrphans(string Folder, IReadOnlyList<string> UnindexedFiles, IReadOnlyList<SidecarFrame> MissingFiles)
|
||||
{
|
||||
public bool IsEmpty => UnindexedFiles.Count == 0 && MissingFiles.Count == 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds and reconciles sidecars from the photos found in a folder (BRIEF §7, §12.5 "Adopt folder"). Only light
|
||||
/// metadata is read; originals are never moved, renamed or rewritten. Hashes are left empty for a verify to fill.
|
||||
/// </summary>
|
||||
public static class SequenceAdopter
|
||||
{
|
||||
/// <summary>A new sidecar for a folder without one. A folder outside the layout (no <c>originals/</c>) is titled by its name.</summary>
|
||||
public static SequenceManifest Build(Guid id, SequencePaths sequence, string? createdBy, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (frames, first) = ReadFrames(sequence, ArchiveScanner.ListPhotos(sequence), cancellationToken);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var inLayout = Directory.Exists(sequence.Originals);
|
||||
var manifest = new SequenceManifest
|
||||
{
|
||||
Id = id,
|
||||
Title = inLayout ? null : Path.GetFileName(Path.TrimEndingDirectorySeparator(sequence.Folder)),
|
||||
Camera = first is null ? null : new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial },
|
||||
Lens = first?.LensModel,
|
||||
CreatedBy = createdBy,
|
||||
Created = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
return SequenceFrames.Insert(manifest, frames);
|
||||
}
|
||||
|
||||
/// <summary>Unindexed photos and missing frame files of one sequence.</summary>
|
||||
public static SequenceOrphans FindOrphans(SequencePaths sequence, SequenceManifest manifest)
|
||||
{
|
||||
var listed = manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal);
|
||||
var unindexed = ArchiveScanner.ListPhotos(sequence).Where(f => !listed.Contains(f)).ToList();
|
||||
var missing = manifest.Frames.Where(f => !File.Exists(sequence.File(f.File))).ToList();
|
||||
return new SequenceOrphans(sequence.RelativeFolder, unindexed, missing);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Brings a sidecar in line with its folder: unindexed photos are added (in capture order), frames whose file is
|
||||
/// gone get <see cref="FrameFlags.Missing"/> and frames whose file came back lose it. Null when nothing changed.
|
||||
/// </summary>
|
||||
public static SequenceManifest? Reconcile(SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var orphans = FindOrphans(sequence, manifest);
|
||||
var missing = orphans.MissingFiles.Select(f => f.Id).ToHashSet();
|
||||
var flagChanges = new Dictionary<Guid, Func<SidecarFrame, SidecarFrame>>();
|
||||
foreach (var frame in manifest.Frames)
|
||||
{
|
||||
var isMissing = missing.Contains(frame.Id);
|
||||
if (isMissing == frame.Flags.HasFlag(FrameFlags.Missing)) continue;
|
||||
flagChanges[frame.Id] = f => f with { Flags = isMissing ? f.Flags | FrameFlags.Missing : f.Flags & ~FrameFlags.Missing };
|
||||
}
|
||||
if (flagChanges.Count == 0 && orphans.UnindexedFiles.Count == 0) return null;
|
||||
var updated = flagChanges.Count == 0 ? manifest : SequenceFrames.Update(manifest, flagChanges);
|
||||
var (added, first) = ReadFrames(sequence, orphans.UnindexedFiles, cancellationToken);
|
||||
if (updated.Camera is null && first is not null)
|
||||
updated = updated with { Camera = new SidecarCamera { Make = first.Make, Model = first.Model, Serial = first.BodySerial }, Lens = updated.Lens ?? first.LensModel };
|
||||
return SequenceFrames.Insert(updated, added);
|
||||
}
|
||||
|
||||
private static (List<SidecarFrame> Frames, FrameMetadata? First) ReadFrames(SequencePaths sequence, IReadOnlyList<string> files, CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = new List<SidecarFrame>(files.Count);
|
||||
FrameMetadata? first = null;
|
||||
foreach (var file in files)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var path = sequence.File(file);
|
||||
try
|
||||
{
|
||||
var metadata = MetadataReader.Read(path, cancellationToken);
|
||||
if (first is null || (metadata.CapturedAt < first.CapturedAt && metadata.Make is not null)) first = metadata;
|
||||
frames.Add(SequenceFrames.FromMetadata(Guid.CreateVersion7(), file, Path.GetFileName(file), new FileInfo(path).Length, null, metadata));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"Skipped unreadable photo {sequence.ArchiveRelative(file)}", ex);
|
||||
}
|
||||
}
|
||||
return (frames, first);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>Edits of the frame list of a <see cref="SequenceManifest"/>; frames stay in capture order with gap-free indexes.</summary>
|
||||
public static class SequenceFrames
|
||||
{
|
||||
/// <summary>Capture time (unknown last), then file name.</summary>
|
||||
public static readonly IComparer<SidecarFrame> CaptureOrder = Comparer<SidecarFrame>.Create((a, b) =>
|
||||
{
|
||||
var byTime = (a.CapturedAt ?? DateTime.MaxValue).CompareTo(b.CapturedAt ?? DateTime.MaxValue);
|
||||
return byTime != 0 ? byTime : string.CompareOrdinal(a.File, b.File);
|
||||
});
|
||||
|
||||
/// <summary>A frame record from light metadata.</summary>
|
||||
/// <param name="file">Path relative to the sequence folder.</param>
|
||||
/// <param name="fallbackCapture">Used when the file records no capture time of its own (only a file-system date).</param>
|
||||
public static SidecarFrame FromMetadata(Guid id, string file, string? original, long size, string? sha256, FrameMetadata metadata,
|
||||
DateTime? fallbackCapture = null)
|
||||
{
|
||||
var useFallback = fallbackCapture is not null && metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown;
|
||||
return new SidecarFrame
|
||||
{
|
||||
Id = id,
|
||||
File = file,
|
||||
Original = original,
|
||||
Size = size,
|
||||
Sha256 = sha256?.ToLowerInvariant(),
|
||||
CapturedAt = useFallback ? DateTime.SpecifyKind(fallbackCapture!.Value, DateTimeKind.Unspecified) : metadata.CapturedAt,
|
||||
UtcOffset = useFallback ? null : metadata.UtcOffsetMinutes,
|
||||
TimeSource = useFallback ? TimestampSource.Unknown : metadata.TimestampSource,
|
||||
Exposure = metadata.ExposureTime,
|
||||
FNumber = metadata.FNumber,
|
||||
Iso = metadata.Iso,
|
||||
Focal = metadata.FocalLength,
|
||||
Bias = metadata.ExposureBias,
|
||||
Wb = metadata.WhiteBalanceMode,
|
||||
Orientation = metadata.Orientation,
|
||||
W = metadata.Width,
|
||||
H = metadata.Height,
|
||||
Format = metadata.Format,
|
||||
PreviewOffset = metadata.LargestPreview?.Offset,
|
||||
PreviewLength = metadata.LargestPreview?.Length,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges <paramref name="added"/> into the frame list by capture time (existing frames keep their relative order),
|
||||
/// renumbers indexes and bumps the frames version.
|
||||
/// </summary>
|
||||
public static SequenceManifest Insert(SequenceManifest manifest, IReadOnlyList<SidecarFrame> added)
|
||||
{
|
||||
if (added.Count == 0) return manifest;
|
||||
var incoming = added.Order(CaptureOrder).ToList();
|
||||
var merged = new List<SidecarFrame>(manifest.Frames.Count + incoming.Count);
|
||||
var next = 0;
|
||||
foreach (var frame in manifest.Frames)
|
||||
{
|
||||
while (next < incoming.Count && CaptureOrder.Compare(incoming[next], frame) < 0) merged.Add(incoming[next++]);
|
||||
merged.Add(frame);
|
||||
}
|
||||
while (next < incoming.Count) merged.Add(incoming[next++]);
|
||||
return WithFrames(manifest, merged);
|
||||
}
|
||||
|
||||
/// <summary>Replaces the frame list (indexes renumbered) and bumps the frames version.</summary>
|
||||
public static SequenceManifest WithFrames(SequenceManifest manifest, IReadOnlyList<SidecarFrame> frames)
|
||||
{
|
||||
var renumbered = new SidecarFrame[frames.Count];
|
||||
for (var i = 0; i < frames.Count; i++) renumbered[i] = frames[i].I == i ? frames[i] : frames[i] with { I = i };
|
||||
return manifest with
|
||||
{
|
||||
Frames = renumbered,
|
||||
Versions = manifest.Versions with { Frames = manifest.Versions.Frames + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>Applies per-frame changes by id (flags, hashes); frames not in <paramref name="changes"/> are kept.</summary>
|
||||
public static SequenceManifest Update(SequenceManifest manifest, IReadOnlyDictionary<Guid, Func<SidecarFrame, SidecarFrame>> changes)
|
||||
{
|
||||
var frames = manifest.Frames.Select(f => changes.TryGetValue(f.Id, out var change) ? change(f) : f).ToList();
|
||||
return manifest with { Frames = frames, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// <c>.titano/sequence.json</c>, the canonical record of one sequence (BRIEF §7). The database is rebuilt from it,
|
||||
/// so it carries the ids of the sequence and of every frame. Fields this build does not know are kept in
|
||||
/// <see cref="Extra"/> and written back unchanged.
|
||||
/// </summary>
|
||||
public sealed record SequenceManifest
|
||||
{
|
||||
public const int CurrentSchemaVersion = 1;
|
||||
|
||||
public int SchemaVersion { get; init; } = CurrentSchemaVersion;
|
||||
public required Guid Id { get; init; }
|
||||
public string? Title { get; init; }
|
||||
public SidecarCamera? Camera { get; init; }
|
||||
public string? Lens { get; init; }
|
||||
|
||||
/// <summary>Frames in capture order; <see cref="SidecarFrame.I"/> equals the position.</summary>
|
||||
public IReadOnlyList<SidecarFrame> Frames { get; init; } = [];
|
||||
|
||||
public IReadOnlyList<string> Tags { get; init; } = [];
|
||||
public int Rating { get; init; }
|
||||
public string? Notes { get; init; }
|
||||
|
||||
/// <summary><see cref="SequenceStatus"/> bits, stored as a number so bits of newer builds survive.</summary>
|
||||
public SequenceStatus Status { get; init; }
|
||||
|
||||
public ManifestVersions Versions { get; init; } = new();
|
||||
|
||||
/// <summary>User id or client name that created the sequence.</summary>
|
||||
public string? CreatedBy { get; init; }
|
||||
|
||||
public DateTimeOffset Created { get; init; }
|
||||
public DateTimeOffset UpdatedAt { get; init; }
|
||||
public DateTimeOffset? VerifiedAt { get; init; }
|
||||
|
||||
/// <summary>Session keys of automatic grouping (uploads, card imports) that resolve to this sequence.</summary>
|
||||
public IReadOnlyList<string> SessionKeys { get; init; } = [];
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
public sealed record SidecarCamera
|
||||
{
|
||||
public string? Make { get; init; }
|
||||
public string? Model { get; init; }
|
||||
public string? Serial { get; init; }
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Counters the sync protocol compares (BRIEF §14.5).</summary>
|
||||
public sealed record ManifestVersions
|
||||
{
|
||||
public int Settings { get; init; }
|
||||
public int Frames { get; init; }
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>One frame of <see cref="SequenceManifest"/>; property names follow BRIEF §7.</summary>
|
||||
public sealed record SidecarFrame
|
||||
{
|
||||
public required Guid Id { get; init; }
|
||||
|
||||
/// <summary>Index in capture order, from 0.</summary>
|
||||
public int I { get; init; }
|
||||
|
||||
/// <summary>Path relative to the sequence folder, forward slashes (<c>originals/…</c> for imported frames).</summary>
|
||||
public required string File { get; init; }
|
||||
|
||||
/// <summary>Name the file had on the card or upload source.</summary>
|
||||
public string? Original { get; init; }
|
||||
|
||||
public long Size { get; init; }
|
||||
|
||||
/// <summary>Lower-case hex SHA-256; null until a verify hashes an adopted file.</summary>
|
||||
public string? Sha256 { get; init; }
|
||||
|
||||
/// <summary>Camera wall-clock time with sub-seconds.</summary>
|
||||
public DateTime? CapturedAt { get; init; }
|
||||
|
||||
public int? UtcOffset { get; init; }
|
||||
|
||||
[JsonConverter(typeof(TolerantEnumConverter<TimestampSource>))]
|
||||
public TimestampSource TimeSource { get; init; }
|
||||
|
||||
public double? Exposure { get; init; }
|
||||
public double? FNumber { get; init; }
|
||||
public int? Iso { get; init; }
|
||||
public double? Focal { get; init; }
|
||||
public double? Bias { get; init; }
|
||||
public int? Wb { get; init; }
|
||||
public int Orientation { get; init; } = 1;
|
||||
public int W { get; init; }
|
||||
public int H { get; init; }
|
||||
|
||||
[JsonConverter(typeof(TolerantEnumConverter<ImageFileFormat>))]
|
||||
public ImageFileFormat Format { get; init; }
|
||||
|
||||
public long? PreviewOffset { get; init; }
|
||||
public long? PreviewLength { get; init; }
|
||||
|
||||
/// <summary><see cref="FrameFlags"/> bits, stored as a number so bits of newer builds survive.</summary>
|
||||
public FrameFlags Flags { get; init; }
|
||||
|
||||
[JsonExtensionData]
|
||||
public Dictionary<string, JsonElement>? Extra { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
using System.Buffers.Binary;
|
||||
using Titano.Core.Hashing;
|
||||
|
||||
namespace Titano.Core.Archiving;
|
||||
|
||||
/// <summary>One thumbnail of a pack: where its JPEG lies and which source it was made from.</summary>
|
||||
/// <param name="SourceHash">32-byte fingerprint of the original (its SHA-256 when known); a mismatch means stale.</param>
|
||||
public sealed record ThumbEntry(int FrameIndex, long Offset, int Length, int Width, int Height, byte[] SourceHash, uint Crc);
|
||||
|
||||
/// <summary>
|
||||
/// <c>thumbs.pack</c>: every thumbnail of a sequence in one file (BRIEF §8.5), so a share holds one file per sequence
|
||||
/// instead of thousands. Little-endian blocks after a 16-byte header:
|
||||
/// <list type="bullet">
|
||||
/// <item><c>TENT</c> entry: frame index, width, height, 32-byte source hash, length, CRC-32, JPEG bytes.</item>
|
||||
/// <item><c>TIDX</c> index: count, one row per frame (latest entry wins), CRC-32 of the rows.</item>
|
||||
/// <item><c>TFTR</c> footer (last 16 bytes): offset of the index block and its CRC.</item>
|
||||
/// </list>
|
||||
/// Entries are never moved, so offsets stay valid for open readers. A writer appends entries where the last index
|
||||
/// began and writes a new index plus footer on commit; after a crash the footer is missing or invalid, readers fall
|
||||
/// back to walking the entry headers, and an incomplete tail is ignored (the writer truncates it).
|
||||
/// </summary>
|
||||
public static class ThumbPack
|
||||
{
|
||||
public const int HashLength = 32;
|
||||
internal const int HeaderLength = 16;
|
||||
internal const int EntryHeaderLength = 4 + 4 + 2 + 2 + HashLength + 4 + 4;
|
||||
internal const int IndexRowLength = 4 + 8 + 4 + 2 + 2 + HashLength + 4;
|
||||
internal const int FooterLength = 16;
|
||||
internal const ushort Version = 1;
|
||||
|
||||
/// <summary>Largest thumbnail accepted; anything bigger in a header is corruption.</summary>
|
||||
internal const int MaxEntryLength = 16 << 20;
|
||||
|
||||
internal static ReadOnlySpan<byte> Magic => "TTPK"u8;
|
||||
internal static ReadOnlySpan<byte> EntryTag => "TENT"u8;
|
||||
internal static ReadOnlySpan<byte> IndexTag => "TIDX"u8;
|
||||
internal static ReadOnlySpan<byte> FooterTag => "TFTR"u8;
|
||||
|
||||
internal static void WriteHeader(Stream stream)
|
||||
{
|
||||
Span<byte> header = stackalloc byte[HeaderLength];
|
||||
header.Clear();
|
||||
Magic.CopyTo(header);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[4..], Version);
|
||||
stream.Write(header);
|
||||
}
|
||||
|
||||
internal static bool HasValidHeader(Stream stream)
|
||||
{
|
||||
if (stream.Length < HeaderLength) return false;
|
||||
Span<byte> header = stackalloc byte[HeaderLength];
|
||||
stream.Position = 0;
|
||||
stream.ReadExactly(header);
|
||||
return header[..4].SequenceEqual(Magic) && BinaryPrimitives.ReadUInt16LittleEndian(header[4..]) == Version;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The latest entry per frame and where the valid content ends: after the footer when the index is intact
|
||||
/// (<paramref name="indexOffset"/> = start of that index), otherwise after the last complete block.
|
||||
/// </summary>
|
||||
internal static Dictionary<int, ThumbEntry> Load(Stream stream, out long validEnd, out long? indexOffset)
|
||||
{
|
||||
indexOffset = null;
|
||||
if (TryLoadIndex(stream, out var entries, out var offset))
|
||||
{
|
||||
validEnd = stream.Length;
|
||||
indexOffset = offset;
|
||||
return entries;
|
||||
}
|
||||
return Scan(stream, out validEnd);
|
||||
}
|
||||
|
||||
private static bool TryLoadIndex(Stream stream, out Dictionary<int, ThumbEntry> entries, out long indexOffset)
|
||||
{
|
||||
entries = [];
|
||||
indexOffset = 0;
|
||||
var length = stream.Length;
|
||||
if (length < HeaderLength + 12 + FooterLength) return false;
|
||||
Span<byte> footer = stackalloc byte[FooterLength];
|
||||
stream.Position = length - FooterLength;
|
||||
stream.ReadExactly(footer);
|
||||
if (!footer[..4].SequenceEqual(FooterTag)) return false;
|
||||
indexOffset = BinaryPrimitives.ReadInt64LittleEndian(footer[4..]);
|
||||
if (Crc32.Compute(footer[4..12]) != BinaryPrimitives.ReadUInt32LittleEndian(footer[12..])) return false;
|
||||
if (indexOffset < HeaderLength || indexOffset > length - FooterLength - 12) return false;
|
||||
|
||||
Span<byte> head = stackalloc byte[8];
|
||||
stream.Position = indexOffset;
|
||||
stream.ReadExactly(head);
|
||||
var count = BinaryPrimitives.ReadInt32LittleEndian(head[4..]);
|
||||
if (!head[..4].SequenceEqual(IndexTag) || count < 0 || indexOffset + 8 + (long)count * IndexRowLength + 4 != length - FooterLength) return false;
|
||||
var rows = new byte[count * IndexRowLength + 4];
|
||||
stream.ReadExactly(rows);
|
||||
if (Crc32.Compute(rows.AsSpan(0, count * IndexRowLength)) != BinaryPrimitives.ReadUInt32LittleEndian(rows.AsSpan(count * IndexRowLength))) return false;
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var row = rows.AsSpan(i * IndexRowLength, IndexRowLength);
|
||||
var entry = new ThumbEntry(
|
||||
BinaryPrimitives.ReadInt32LittleEndian(row),
|
||||
BinaryPrimitives.ReadInt64LittleEndian(row[4..]),
|
||||
BinaryPrimitives.ReadInt32LittleEndian(row[12..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(row[16..]),
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(row[18..]),
|
||||
row.Slice(20, HashLength).ToArray(),
|
||||
BinaryPrimitives.ReadUInt32LittleEndian(row[(20 + HashLength)..]));
|
||||
if (entry.Offset < HeaderLength || entry.Length < 0 || entry.Offset + entry.Length > indexOffset) return false;
|
||||
entries[entry.FrameIndex] = entry;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Walks block headers from the start; stops at the first incomplete or unknown block.</summary>
|
||||
private static Dictionary<int, ThumbEntry> Scan(Stream stream, out long validEnd)
|
||||
{
|
||||
var entries = new Dictionary<int, ThumbEntry>();
|
||||
var length = stream.Length;
|
||||
var position = (long)HeaderLength;
|
||||
var header = new byte[EntryHeaderLength];
|
||||
while (position + 4 <= length)
|
||||
{
|
||||
stream.Position = position;
|
||||
stream.ReadExactly(header.AsSpan(0, 4));
|
||||
var tag = header.AsSpan(0, 4);
|
||||
long next;
|
||||
if (tag.SequenceEqual(EntryTag))
|
||||
{
|
||||
if (position + EntryHeaderLength > length) break;
|
||||
stream.ReadExactly(header.AsSpan(4, EntryHeaderLength - 4));
|
||||
var size = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(12 + HashLength));
|
||||
if (size is < 0 or > MaxEntryLength) break;
|
||||
next = position + EntryHeaderLength + size;
|
||||
if (next > length) break;
|
||||
var frame = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
|
||||
entries[frame] = new ThumbEntry(frame, position + EntryHeaderLength, size,
|
||||
BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(8)), BinaryPrimitives.ReadUInt16LittleEndian(header.AsSpan(10)),
|
||||
header.AsSpan(12, HashLength).ToArray(), BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(16 + HashLength)));
|
||||
}
|
||||
else if (tag.SequenceEqual(IndexTag))
|
||||
{
|
||||
if (position + 8 > length) break;
|
||||
stream.ReadExactly(header.AsSpan(4, 4));
|
||||
var count = BinaryPrimitives.ReadInt32LittleEndian(header.AsSpan(4));
|
||||
if (count < 0) break;
|
||||
next = position + 8 + (long)count * IndexRowLength + 4;
|
||||
if (next > length) break;
|
||||
}
|
||||
else if (tag.SequenceEqual(FooterTag))
|
||||
{
|
||||
next = position + FooterLength;
|
||||
if (next > length) break;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
position = next;
|
||||
}
|
||||
validEnd = position;
|
||||
return entries;
|
||||
}
|
||||
|
||||
/// <summary>Writes one entry block at the stream position and returns its entry.</summary>
|
||||
internal static ThumbEntry WriteEntry(Stream stream, int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
|
||||
{
|
||||
var crc = Crc32.Compute(jpeg);
|
||||
Span<byte> header = stackalloc byte[EntryHeaderLength];
|
||||
EntryTag.CopyTo(header);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header[4..], frameIndex);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[8..], (ushort)width);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(header[10..], (ushort)height);
|
||||
sourceHash.CopyTo(header[12..]);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(header[(12 + HashLength)..], jpeg.Length);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(header[(16 + HashLength)..], crc);
|
||||
var payload = stream.Position + EntryHeaderLength;
|
||||
stream.Write(header);
|
||||
stream.Write(jpeg);
|
||||
return new ThumbEntry(frameIndex, payload, jpeg.Length, width, height, sourceHash, crc);
|
||||
}
|
||||
|
||||
internal static void WriteIndexAndFooter(Stream stream, IReadOnlyCollection<ThumbEntry> entries)
|
||||
{
|
||||
var indexOffset = stream.Position;
|
||||
var block = new byte[8 + entries.Count * IndexRowLength + 4];
|
||||
IndexTag.CopyTo(block);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(block.AsSpan(4), entries.Count);
|
||||
var i = 0;
|
||||
foreach (var e in entries.OrderBy(e => e.FrameIndex))
|
||||
{
|
||||
var row = block.AsSpan(8 + i++ * IndexRowLength, IndexRowLength);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(row, e.FrameIndex);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(row[4..], e.Offset);
|
||||
BinaryPrimitives.WriteInt32LittleEndian(row[12..], e.Length);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(row[16..], (ushort)e.Width);
|
||||
BinaryPrimitives.WriteUInt16LittleEndian(row[18..], (ushort)e.Height);
|
||||
e.SourceHash.CopyTo(row[20..]);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(row[(20 + HashLength)..], e.Crc);
|
||||
}
|
||||
var rowsLength = entries.Count * IndexRowLength;
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(block.AsSpan(8 + rowsLength), Crc32.Compute(block.AsSpan(8, rowsLength)));
|
||||
stream.Write(block);
|
||||
|
||||
Span<byte> footer = stackalloc byte[FooterLength];
|
||||
FooterTag.CopyTo(footer);
|
||||
BinaryPrimitives.WriteInt64LittleEndian(footer[4..], indexOffset);
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(footer[12..], Crc32.Compute(footer[4..12]));
|
||||
stream.Write(footer);
|
||||
}
|
||||
|
||||
/// <summary>Size of a pack holding only <paramref name="entries"/>; the rest of a file is reclaimable garbage.</summary>
|
||||
public static long LiveBytes(IEnumerable<ThumbEntry> entries)
|
||||
{
|
||||
long count = 0, payload = 0;
|
||||
foreach (var e in entries)
|
||||
{
|
||||
count++;
|
||||
payload += EntryHeaderLength + e.Length;
|
||||
}
|
||||
return HeaderLength + payload + 8 + count * IndexRowLength + 4 + FooterLength;
|
||||
}
|
||||
|
||||
/// <summary>A 32-byte source fingerprint: the SHA-256 itself when the original's hash is known.</summary>
|
||||
public static byte[] Fingerprint(string? sha256, string file, long size) =>
|
||||
sha256 is { Length: HashLength * 2 }
|
||||
? Convert.FromHexString(sha256)
|
||||
: System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes($"{file}|{size}"));
|
||||
}
|
||||
|
||||
/// <summary>Random access to the thumbnails of a pack without loading the file; safe while a writer appends.</summary>
|
||||
public sealed class ThumbPackReader : IDisposable
|
||||
{
|
||||
private readonly FileStream? _stream;
|
||||
private readonly Lock _gate = new();
|
||||
|
||||
private ThumbPackReader(FileStream? stream, Dictionary<int, ThumbEntry> entries)
|
||||
{
|
||||
_stream = stream;
|
||||
Entries = entries;
|
||||
}
|
||||
|
||||
/// <summary>The latest entry of every frame that has one.</summary>
|
||||
public IReadOnlyDictionary<int, ThumbEntry> Entries { get; }
|
||||
|
||||
/// <summary>Bytes held by replaced entries (reclaimed by <see cref="ThumbPackWriter.Compact"/>).</summary>
|
||||
public long GarbageBytes => _stream is null ? 0 : System.Math.Max(0, _stream.Length - ThumbPack.LiveBytes(Entries.Values));
|
||||
|
||||
public long Length => _stream?.Length ?? 0;
|
||||
|
||||
/// <summary>Opens a pack; a missing or damaged pack reads as empty.</summary>
|
||||
public static ThumbPackReader Open(string path)
|
||||
{
|
||||
if (!File.Exists(path)) return new ThumbPackReader(null, []);
|
||||
var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, 4096, FileOptions.RandomAccess);
|
||||
try
|
||||
{
|
||||
var entries = ThumbPack.HasValidHeader(stream) ? ThumbPack.Load(stream, out _, out _) : [];
|
||||
return new ThumbPackReader(stream, entries);
|
||||
}
|
||||
catch
|
||||
{
|
||||
stream.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The JPEG of a frame, or null when absent or damaged (CRC mismatch).</summary>
|
||||
public byte[]? Read(int frameIndex)
|
||||
{
|
||||
if (_stream is null || !Entries.TryGetValue(frameIndex, out var entry)) return null;
|
||||
var data = new byte[entry.Length];
|
||||
lock (_gate)
|
||||
{
|
||||
if (entry.Offset + entry.Length > _stream.Length) return null;
|
||||
_stream.Position = entry.Offset;
|
||||
_stream.ReadExactly(data);
|
||||
}
|
||||
return Crc32.Compute(data) == entry.Crc ? data : null;
|
||||
}
|
||||
|
||||
public void Dispose() => _stream?.Dispose();
|
||||
}
|
||||
|
||||
/// <summary>Appends thumbnails to a pack; one writer per pack at a time (the sequence lease guarantees it).</summary>
|
||||
public sealed class ThumbPackWriter : IDisposable
|
||||
{
|
||||
private readonly FileStream _stream;
|
||||
private readonly Dictionary<int, ThumbEntry> _entries;
|
||||
private long? _indexOffset;
|
||||
private bool _dirty;
|
||||
|
||||
private ThumbPackWriter(FileStream stream, Dictionary<int, ThumbEntry> entries, long? indexOffset)
|
||||
{
|
||||
_stream = stream;
|
||||
_entries = entries;
|
||||
_indexOffset = indexOffset;
|
||||
}
|
||||
|
||||
public IReadOnlyDictionary<int, ThumbEntry> Entries => _entries;
|
||||
|
||||
/// <summary>Opens or creates a pack, dropping an incomplete tail left by a crash.</summary>
|
||||
public static ThumbPackWriter Open(string path)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(path))!);
|
||||
var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read | FileShare.Delete, 1 << 16);
|
||||
try
|
||||
{
|
||||
if (!ThumbPack.HasValidHeader(stream))
|
||||
{
|
||||
// Not a pack (or a header torn by a crash): thumbnails are regenerable, start over.
|
||||
stream.SetLength(0);
|
||||
ThumbPack.WriteHeader(stream);
|
||||
return new ThumbPackWriter(stream, [], null);
|
||||
}
|
||||
var entries = ThumbPack.Load(stream, out var validEnd, out var indexOffset);
|
||||
if (validEnd < stream.Length) stream.SetLength(validEnd);
|
||||
return new ThumbPackWriter(stream, entries, indexOffset);
|
||||
}
|
||||
catch
|
||||
{
|
||||
stream.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Appends a thumbnail; it replaces any earlier entry of the frame once committed.</summary>
|
||||
public void Append(int frameIndex, int width, int height, byte[] sourceHash, ReadOnlySpan<byte> jpeg)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfNegative(frameIndex);
|
||||
if (sourceHash.Length != ThumbPack.HashLength) throw new ArgumentException("A source hash has 32 bytes.", nameof(sourceHash));
|
||||
if (jpeg.Length > ThumbPack.MaxEntryLength) throw new ArgumentException("Thumbnail too large.", nameof(jpeg));
|
||||
if (width is < 1 or > ushort.MaxValue || height is < 1 or > ushort.MaxValue) throw new ArgumentOutOfRangeException(nameof(width));
|
||||
// The last committed index sits at the end; new entries overwrite it (readers fall back to a header walk).
|
||||
_stream.Position = _indexOffset ?? _stream.Length;
|
||||
if (_indexOffset is { } offset) _stream.SetLength(offset);
|
||||
_indexOffset = null;
|
||||
|
||||
_entries[frameIndex] = ThumbPack.WriteEntry(_stream, frameIndex, width, height, sourceHash, jpeg);
|
||||
_dirty = true;
|
||||
}
|
||||
|
||||
/// <summary>Writes the index and footer and flushes to disk.</summary>
|
||||
public void Commit()
|
||||
{
|
||||
if (!_dirty) return;
|
||||
_stream.Position = _stream.Length;
|
||||
var indexOffset = _stream.Position;
|
||||
ThumbPack.WriteIndexAndFooter(_stream, _entries.Values);
|
||||
_stream.Flush(flushToDisk: true);
|
||||
_indexOffset = indexOffset;
|
||||
_dirty = false;
|
||||
}
|
||||
|
||||
/// <summary>Rewrites the pack with only its live entries (atomically, so readers keep a consistent file).</summary>
|
||||
public static void Compact(string path)
|
||||
{
|
||||
var temp = IO.AtomicFile.TempPathFor(path);
|
||||
try
|
||||
{
|
||||
using (var source = ThumbPackReader.Open(path))
|
||||
using (var target = new FileStream(temp, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 16))
|
||||
{
|
||||
ThumbPack.WriteHeader(target);
|
||||
var entries = new List<ThumbEntry>(source.Entries.Count);
|
||||
foreach (var entry in source.Entries.Values.OrderBy(e => e.FrameIndex))
|
||||
{
|
||||
if (source.Read(entry.FrameIndex) is { } jpeg)
|
||||
entries.Add(ThumbPack.WriteEntry(target, entry.FrameIndex, entry.Width, entry.Height, entry.SourceHash, jpeg));
|
||||
}
|
||||
ThumbPack.WriteIndexAndFooter(target, entries);
|
||||
}
|
||||
IO.AtomicFile.Commit(temp, path);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temp)) File.Delete(temp);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
{
|
||||
Commit();
|
||||
}
|
||||
finally
|
||||
{
|
||||
_stream.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>A card file that was imported, with the hash recorded at import and where its copy lives.</summary>
|
||||
/// <param name="SourcePath">Path below the card root, forward slashes.</param>
|
||||
/// <param name="CopyPath">Absolute path of the archive (or folder) copy.</param>
|
||||
public sealed record EraseCandidate(string SourcePath, long Size, string Sha256, string CopyPath);
|
||||
|
||||
public enum EraseVerdict
|
||||
{
|
||||
/// <summary>Card file and copy both hash to the recorded SHA-256: safe to delete from the card.</summary>
|
||||
Delete,
|
||||
|
||||
SourceMissing,
|
||||
|
||||
/// <summary>The card file no longer matches what was imported.</summary>
|
||||
SourceChanged,
|
||||
|
||||
CopyMissing,
|
||||
|
||||
/// <summary>The copy no longer matches: the card file is the only good one left.</summary>
|
||||
CopyMismatch,
|
||||
}
|
||||
|
||||
/// <param name="SourceModifiedUtc">Card file time when checked; execution refuses a file touched since.</param>
|
||||
public sealed record EraseItem(EraseCandidate Candidate, EraseVerdict Verdict, DateTime? SourceModifiedUtc);
|
||||
|
||||
/// <summary>The dry run of "delete imported" (BRIEF §12.4, §13): exactly the files that would be deleted, and why
|
||||
/// the others are kept.</summary>
|
||||
public sealed record ErasePlan(string SourceRoot, IReadOnlyList<EraseItem> Items)
|
||||
{
|
||||
public IEnumerable<EraseItem> ToDelete => Items.Where(i => i.Verdict == EraseVerdict.Delete);
|
||||
}
|
||||
|
||||
public sealed record EraseResult(int Deleted, IReadOnlyList<EraseItem> Kept);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes imported photos from a card only after re-hashing both the card file and its copy against the hash
|
||||
/// recorded at import (BRIEF §0.2 rule 8). Never formats (that is the OS dialog) and never touches folders.
|
||||
/// </summary>
|
||||
public static class CardEraser
|
||||
{
|
||||
public static async Task<ErasePlan> PlanAsync(string sourceRoot, IReadOnlyList<EraseCandidate> candidates, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = new List<EraseItem>(candidates.Count);
|
||||
for (var i = 0; i < candidates.Count; i++)
|
||||
{
|
||||
var c = candidates[i];
|
||||
var source = new FileInfo(Path.Combine(sourceRoot, c.SourcePath));
|
||||
var verdict = !source.Exists ? EraseVerdict.SourceMissing
|
||||
: !File.Exists(c.CopyPath) ? EraseVerdict.CopyMissing
|
||||
: !await MatchesAsync(c.CopyPath, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.CopyMismatch
|
||||
: !await MatchesAsync(source.FullName, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.SourceChanged
|
||||
: EraseVerdict.Delete;
|
||||
items.Add(new EraseItem(c, verdict, source.Exists ? source.LastWriteTimeUtc : null));
|
||||
progress?.Report(i + 1);
|
||||
}
|
||||
return new ErasePlan(Path.GetFullPath(sourceRoot), items);
|
||||
}
|
||||
|
||||
/// <summary>Deletes the plan's <see cref="EraseVerdict.Delete"/> files whose size and time are unchanged since planning.</summary>
|
||||
public static EraseResult Execute(ErasePlan plan, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var deleted = 0;
|
||||
var kept = plan.Items.Where(i => i.Verdict != EraseVerdict.Delete).ToList();
|
||||
foreach (var item in plan.ToDelete)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var source = new FileInfo(Path.Combine(plan.SourceRoot, item.Candidate.SourcePath));
|
||||
var copy = new FileInfo(item.Candidate.CopyPath);
|
||||
if (!source.Exists || source.Length != item.Candidate.Size || source.LastWriteTimeUtc != item.SourceModifiedUtc
|
||||
|| !copy.Exists || copy.Length != item.Candidate.Size)
|
||||
{
|
||||
kept.Add(item with { Verdict = EraseVerdict.SourceChanged });
|
||||
continue;
|
||||
}
|
||||
source.Delete();
|
||||
deleted++;
|
||||
Log.Info("import", $"Deleted verified {item.Candidate.SourcePath} from the card");
|
||||
}
|
||||
return new EraseResult(deleted, kept);
|
||||
}
|
||||
|
||||
private static async Task<bool> MatchesAsync(string path, EraseCandidate candidate, CancellationToken cancellationToken) =>
|
||||
new FileInfo(path).Length == candidate.Size
|
||||
&& string.Equals(await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false), candidate.Sha256, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Metadata;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>A photo file on a card or source folder.</summary>
|
||||
/// <param name="RelativePath">Path below the source root, forward slashes (the card import history key).</param>
|
||||
public sealed record CardFile(string RelativePath, long Size, DateTime ModifiedUtc);
|
||||
|
||||
/// <summary>A card photo with its light metadata.</summary>
|
||||
public sealed record CardPhoto(CardFile File, FrameMetadata Metadata)
|
||||
{
|
||||
/// <summary>Capture time, or the file time when the photo records none.</summary>
|
||||
public DateTime CaptureTime => Metadata.CapturedAt ?? File.ModifiedUtc;
|
||||
}
|
||||
|
||||
/// <summary>A file that could not be read while planning; it is listed, never a reason to stop.</summary>
|
||||
public sealed record ImportProblem(string RelativePath, string Message);
|
||||
|
||||
/// <summary>Finds photos on a card (BRIEF §12.4, §13): the <c>DCIM</c> tree when present, otherwise the root.</summary>
|
||||
public static class CardScanner
|
||||
{
|
||||
private const int MaxDepth = 12;
|
||||
|
||||
private static readonly EnumerationOptions Listing = new()
|
||||
{
|
||||
IgnoreInaccessible = true,
|
||||
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System | FileAttributes.Hidden,
|
||||
RecurseSubdirectories = false,
|
||||
MatchCasing = MatchCasing.CaseInsensitive,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Photos below <paramref name="root"/>: everything under <c>DCIM</c> (camera folders) when the card has one,
|
||||
/// otherwise the root's photos and, with <paramref name="includeSubfolders"/>, its subfolders'. Ordered by path.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<CardFile> Find(string root, bool includeSubfolders, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var dcim = Directory.EnumerateDirectories(root, "DCIM", Listing).FirstOrDefault();
|
||||
var start = dcim ?? root;
|
||||
var recurse = dcim is not null || includeSubfolders;
|
||||
var files = new List<CardFile>();
|
||||
var pending = new Stack<(string Path, int Depth)>();
|
||||
pending.Push((start, 0));
|
||||
while (pending.Count > 0)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var (directory, depth) = pending.Pop();
|
||||
foreach (var file in new DirectoryInfo(directory).EnumerateFiles("*", Listing))
|
||||
{
|
||||
if (ArchiveScanner.IsPhoto(file.Name))
|
||||
files.Add(new CardFile(Path.GetRelativePath(root, file.FullName).Replace('\\', '/'), file.Length, file.LastWriteTimeUtc));
|
||||
}
|
||||
if (!recurse || depth >= MaxDepth) continue;
|
||||
foreach (var child in Directory.EnumerateDirectories(directory, "*", Listing))
|
||||
{
|
||||
if (!Path.GetFileName(child).StartsWith('.')) pending.Push((child, depth + 1));
|
||||
}
|
||||
}
|
||||
files.Sort((a, b) => string.CompareOrdinal(a.RelativePath, b.RelativePath));
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>Reads light metadata of every file; unreadable files become <paramref name="problems"/>.</summary>
|
||||
public static IReadOnlyList<CardPhoto> ReadMetadata(string root, IReadOnlyList<CardFile> files, List<ImportProblem> problems,
|
||||
IProgress<int>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var photos = new List<CardPhoto>(files.Count);
|
||||
for (var i = 0; i < files.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var file = files[i];
|
||||
try
|
||||
{
|
||||
photos.Add(new CardPhoto(file, MetadataReader.Read(Path.Combine(root, file.RelativePath), cancellationToken)));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("import", $"Cannot read {file.RelativePath}", ex);
|
||||
problems.Add(new ImportProblem(file.RelativePath, ex.Message));
|
||||
}
|
||||
progress?.Report(i + 1);
|
||||
}
|
||||
return photos;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Titano.Core.Archiving;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Import destination "Folder" (BRIEF §12.4): photos go to <c>root/<folder>/<name></c> without sidecars or index.
|
||||
/// A name already taken by identical content is a duplicate; by other content, the photo gets a suffix. Nothing at the
|
||||
/// destination is ever replaced.
|
||||
/// </summary>
|
||||
public sealed class FolderImportSink(string root) : IImportSink
|
||||
{
|
||||
public string StagingDirectory { get; } = Path.Combine(Path.GetFullPath(root), ArchiveLayout.MetaFolder, ArchiveLayout.StagingFolder);
|
||||
|
||||
public async Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = Path.Combine(Path.GetFullPath(root), session.Folder.Replace('/', Path.DirectorySeparatorChar));
|
||||
Directory.CreateDirectory(folder);
|
||||
var results = new List<SinkCommit>(files.Count);
|
||||
foreach (var file in files)
|
||||
{
|
||||
var name = file.Item.FileName;
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
var duplicate = false;
|
||||
for (var n = 2; File.Exists(Path.Combine(folder, name)); n++)
|
||||
{
|
||||
if (await IsSameAsync(Path.Combine(folder, name), file, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
duplicate = true;
|
||||
break;
|
||||
}
|
||||
name = $"{stem}_{n}{Path.GetExtension(file.Item.FileName)}";
|
||||
}
|
||||
if (!duplicate) File.Move(file.StagedPath, Path.Combine(folder, name), overwrite: false);
|
||||
results.Add(new SinkCommit(null, session.Folder + "/" + name, duplicate));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static async Task<bool> IsSameAsync(string existing, StagedFile file, CancellationToken cancellationToken) =>
|
||||
new FileInfo(existing).Length == file.Item.Size
|
||||
&& await ContentHash.Sha256Async(existing, null, cancellationToken).ConfigureAwait(false) == file.Sha256;
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>The card or source folder disappeared mid-import (card removed, share lost). The import can resume.</summary>
|
||||
public sealed class ImportSourceUnavailableException(string root, Exception? inner = null)
|
||||
: IOException(Loc.L("Archive.Error.SourceGone", root), inner)
|
||||
{
|
||||
public string Root { get; } = root;
|
||||
}
|
||||
|
||||
/// <summary>A verified copy waiting in the sink's staging folder.</summary>
|
||||
public sealed record StagedFile(PlannedItem Item, string StagedPath, string Sha256);
|
||||
|
||||
/// <summary>Where a staged file ended up.</summary>
|
||||
/// <param name="FrameId">The archive frame (null for a plain folder destination).</param>
|
||||
/// <param name="Duplicate">The destination already held this content; the staged copy was not used.</param>
|
||||
public sealed record SinkCommit(Guid? FrameId, string RelativePath, bool Duplicate);
|
||||
|
||||
/// <summary>Destination of an import: the archive, a plain folder, or a server (BRIEF §12.4).</summary>
|
||||
public interface IImportSink
|
||||
{
|
||||
/// <summary>Folder for verified copies, on the destination volume so commits are renames.</summary>
|
||||
string StagingDirectory { get; }
|
||||
|
||||
/// <summary>Moves staged files of one session into place; returns one result per file, in order. Files it does
|
||||
/// not move stay where they are and the executor deletes them.</summary>
|
||||
Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public enum ImportItemStatus
|
||||
{
|
||||
Copied,
|
||||
|
||||
/// <summary>Not copied: imported before, or already in the archive.</summary>
|
||||
Skipped,
|
||||
|
||||
/// <summary>Copied and verified, but the destination already held the same content.</summary>
|
||||
Duplicate,
|
||||
|
||||
Failed,
|
||||
}
|
||||
|
||||
public sealed record ImportItemResult(PlannedSession Session, PlannedItem Item, ImportItemStatus Status, string? Sha256, SinkCommit? Commit, string? Error);
|
||||
|
||||
/// <summary>Live figures for the job bar: items and bytes (throughput and ETA come from bytes).</summary>
|
||||
public sealed record ImportProgress(int ItemsDone, int ItemsTotal, long BytesDone, long BytesTotal, int Copied, int Skipped, int Failed, string? Current);
|
||||
|
||||
/// <summary>Results committed since the previous batch; <see cref="NextItem"/> is the resume point once they are recorded.</summary>
|
||||
public sealed record ImportBatch(IReadOnlyList<ImportItemResult> Results, int NextItem, ImportProgress Progress);
|
||||
|
||||
/// <summary>
|
||||
/// Executes an <see cref="ImportPlan"/> (BRIEF §12.4, §13): copies each photo to staging with hashing and read-back
|
||||
/// verification, commits verified copies to the sink in batches, reports per-file errors without stopping, and can
|
||||
/// resume from any batch boundary. When the source disappears it commits what is already verified and throws
|
||||
/// <see cref="ImportSourceUnavailableException"/>.
|
||||
/// </summary>
|
||||
public sealed class ImportExecutor(ImportPlan plan, IImportSink sink)
|
||||
{
|
||||
private const int BatchSize = 32;
|
||||
|
||||
/// <param name="startAt">Number of plan items already done (the last <see cref="ImportBatch.NextItem"/> recorded).</param>
|
||||
/// <param name="committed">Called after each batch; record the results, then checkpoint <see cref="ImportBatch.NextItem"/>.</param>
|
||||
/// <param name="progress">Called before and after each copy (throughput and ETA come from the bytes).</param>
|
||||
public async Task<ImportProgress> RunAsync(int startAt, bool verify, Func<ImportBatch, ValueTask> committed,
|
||||
Func<ImportProgress, ValueTask>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = plan.Sessions.SelectMany(s => s.Items.Select(i => (Session: s, Item: i))).ToList();
|
||||
var totalBytes = items.Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size);
|
||||
var bytesDone = items.Take(startAt).Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size);
|
||||
int copied = 0, skipped = 0, failed = 0;
|
||||
var results = new List<ImportItemResult>();
|
||||
var staged = new List<StagedFile>();
|
||||
PlannedSession? stagedSession = null;
|
||||
var next = startAt;
|
||||
|
||||
ImportProgress Snapshot(string? current) => new(next, items.Count, bytesDone, totalBytes, copied, skipped, failed, current);
|
||||
|
||||
async Task FlushAsync()
|
||||
{
|
||||
if (staged.Count > 0)
|
||||
{
|
||||
var commits = await sink.CommitAsync(stagedSession!, staged, cancellationToken).ConfigureAwait(false);
|
||||
for (var i = 0; i < staged.Count; i++)
|
||||
{
|
||||
var duplicate = commits[i].Duplicate;
|
||||
results.Add(new ImportItemResult(stagedSession!, staged[i].Item, duplicate ? ImportItemStatus.Duplicate : ImportItemStatus.Copied,
|
||||
staged[i].Sha256, commits[i], null));
|
||||
if (duplicate) skipped++;
|
||||
else copied++;
|
||||
DeleteLeftover(staged[i].StagedPath);
|
||||
}
|
||||
staged.Clear();
|
||||
stagedSession = null;
|
||||
}
|
||||
if (results.Count == 0) return;
|
||||
await committed(new ImportBatch([.. results], next, Snapshot(null))).ConfigureAwait(false);
|
||||
results.Clear();
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(sink.StagingDirectory);
|
||||
for (; next < items.Count; next++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var (session, item) = items[next];
|
||||
if ((stagedSession is not null && !ReferenceEquals(stagedSession, session)) || staged.Count + results.Count >= BatchSize)
|
||||
await FlushAsync().ConfigureAwait(false);
|
||||
if (item.Action != ImportAction.Copy)
|
||||
{
|
||||
skipped++;
|
||||
results.Add(new ImportItemResult(session, item, ImportItemStatus.Skipped, item.Sha256, null, null));
|
||||
continue;
|
||||
}
|
||||
var source = Path.Combine(plan.SourceRoot, item.SourcePath);
|
||||
if (progress is not null) await progress(Snapshot(item.SourcePath)).ConfigureAwait(false);
|
||||
var target = Path.Combine(sink.StagingDirectory, $"{plan.Id:N}-{next:D6}{Path.GetExtension(item.FileName)}");
|
||||
DeleteLeftover(target);
|
||||
try
|
||||
{
|
||||
if (new FileInfo(source) is not { Exists: true } info || info.Length != item.Size)
|
||||
throw new IOException(Loc.L("Archive.Error.SourceChanged", item.SourcePath));
|
||||
var sha = await VerifiedCopy.CopyAsync(source, target, verify, read => bytesDone += read, cancellationToken).ConfigureAwait(false);
|
||||
staged.Add(new StagedFile(item, target, sha));
|
||||
stagedSession = session;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
if (!Directory.Exists(plan.SourceRoot))
|
||||
{
|
||||
await FlushAsync().ConfigureAwait(false);
|
||||
throw new ImportSourceUnavailableException(plan.SourceRoot, ex);
|
||||
}
|
||||
Log.Warning("import", $"Copy of {item.SourcePath} failed", ex);
|
||||
failed++;
|
||||
results.Add(new ImportItemResult(session, item, ImportItemStatus.Failed, null, null, ex.Message));
|
||||
}
|
||||
}
|
||||
await FlushAsync().ConfigureAwait(false);
|
||||
return Snapshot(null);
|
||||
}
|
||||
|
||||
private static void DeleteLeftover(string staged)
|
||||
{
|
||||
// Staged files are this import's own copies of card files; the originals stay on the card.
|
||||
if (File.Exists(staged)) File.Delete(staged);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>What to do with a photo that is already in the archive (same SHA-256).</summary>
|
||||
public enum DuplicatePolicy
|
||||
{
|
||||
/// <summary>Leave it on the card; the archive copy is identical.</summary>
|
||||
Skip,
|
||||
|
||||
/// <summary>Import it again into this session's sequence (its name gets a suffix if taken).</summary>
|
||||
ImportAgain,
|
||||
}
|
||||
|
||||
/// <summary>Card import settings (BRIEF §12.4, §13); the defaults are the brief's.</summary>
|
||||
public sealed record ImportOptions
|
||||
{
|
||||
public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
|
||||
public string FileTemplate { get; init; } = TemplateDefaults.File;
|
||||
public TimeSpan SessionGap { get; init; } = TimeSpan.FromMinutes(45);
|
||||
public int MinimumSessionFrames { get; init; } = 8;
|
||||
public bool IncludeSubfolders { get; init; } = true;
|
||||
public bool VerifyCopies { get; init; } = true;
|
||||
public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
|
||||
|
||||
/// <summary>Card paths where the user split a session (a new session starts at that photo).</summary>
|
||||
public IReadOnlySet<string> SplitBefore { get; init; } = new HashSet<string>();
|
||||
|
||||
/// <summary>Card paths where the user merged a session into the previous one.</summary>
|
||||
public IReadOnlySet<string> MergeAt { get; init; } = new HashSet<string>();
|
||||
|
||||
/// <summary>Titles given to sessions, by session key; used as the sequence title and the <c>{title}</c> token.</summary>
|
||||
public IReadOnlyDictionary<string, string> Titles { get; init; } = new Dictionary<string, string>();
|
||||
}
|
||||
|
||||
public enum ImportAction
|
||||
{
|
||||
Copy,
|
||||
|
||||
/// <summary>Imported from this card before (same path, size and time).</summary>
|
||||
SkipImported,
|
||||
|
||||
/// <summary>Already in the archive (same size and capture time, then the same SHA-256).</summary>
|
||||
SkipInArchive,
|
||||
}
|
||||
|
||||
/// <summary>One photo of the plan and its destination name.</summary>
|
||||
public sealed record PlannedItem
|
||||
{
|
||||
/// <summary>Path below the source root, forward slashes.</summary>
|
||||
public required string SourcePath { get; init; }
|
||||
|
||||
public long Size { get; init; }
|
||||
public DateTime ModifiedUtc { get; init; }
|
||||
public DateTime CapturedAt { get; init; }
|
||||
public required string FileName { get; init; }
|
||||
public ImportAction Action { get; init; }
|
||||
|
||||
/// <summary>The archive frame this photo duplicates (<see cref="ImportAction.SkipInArchive"/>, or imported again).</summary>
|
||||
public Guid? ExistingFrameId { get; init; }
|
||||
|
||||
/// <summary>SHA-256 computed while planning (duplicate check), if any.</summary>
|
||||
public string? Sha256 { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>A session of the plan: one camera body and file format, one sequence in the archive.</summary>
|
||||
public sealed record PlannedSession
|
||||
{
|
||||
/// <summary>1-based, in order of start time across the import.</summary>
|
||||
public int Number { get; init; }
|
||||
|
||||
/// <summary>Stable key of the session; re-planning the same photos yields the same key, so an interrupted import
|
||||
/// resumes into the same sequence.</summary>
|
||||
public required string SessionKey { get; init; }
|
||||
|
||||
public string? CameraMake { get; init; }
|
||||
public string? CameraModel { get; init; }
|
||||
public string? CameraSerial { get; init; }
|
||||
public ImageFileFormat Format { get; init; }
|
||||
public DateTime Start { get; init; }
|
||||
public DateTime End { get; init; }
|
||||
|
||||
/// <summary>Destination folder relative to the destination root, forward slashes.</summary>
|
||||
public required string Folder { get; init; }
|
||||
|
||||
public string? Title { get; init; }
|
||||
public IReadOnlyList<PlannedItem> Items { get; init; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The exact result of an import before any byte is written (BRIEF §12.4 "Plan before touching disk"): sessions,
|
||||
/// folders, file names, what is skipped and why, and the space it needs. Executing it copies exactly this.
|
||||
/// </summary>
|
||||
public sealed record ImportPlan
|
||||
{
|
||||
public required Guid Id { get; init; }
|
||||
public required string SourceRoot { get; init; }
|
||||
public required string DestinationRoot { get; init; }
|
||||
public DateTimeOffset Created { get; init; }
|
||||
public IReadOnlyList<PlannedSession> Sessions { get; init; } = [];
|
||||
public IReadOnlyList<ImportProblem> Problems { get; init; } = [];
|
||||
|
||||
/// <summary>Free space at the destination when planned; null when the volume does not report it.</summary>
|
||||
public long? AvailableBytes { get; init; }
|
||||
|
||||
[JsonIgnore]
|
||||
public IEnumerable<PlannedItem> Items => Sessions.SelectMany(s => s.Items);
|
||||
[JsonIgnore]
|
||||
public int FileCount => Sessions.Sum(s => s.Items.Count);
|
||||
[JsonIgnore]
|
||||
public int CopyCount => Items.Count(i => i.Action == ImportAction.Copy);
|
||||
[JsonIgnore]
|
||||
public long CopyBytes => Items.Where(i => i.Action == ImportAction.Copy).Sum(i => i.Size);
|
||||
[JsonIgnore]
|
||||
public bool HasEnoughSpace => AvailableBytes is not { } free || free >= CopyBytes;
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>An archive frame that may be the same photo.</summary>
|
||||
public sealed record ArchiveMatch(Guid FrameId, string? Sha256);
|
||||
|
||||
/// <summary>What the planner needs to know about the destination (implemented over the archive index).</summary>
|
||||
public interface IImportLookup
|
||||
{
|
||||
/// <summary>The file was imported from this card before (same path, size and time).</summary>
|
||||
Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Archive frames with this size and capture time; the planner compares hashes only for these.</summary>
|
||||
Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Folder of the sequence an earlier import of this session went to, so a re-import continues it.</summary>
|
||||
Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>The folder is taken by something else (another sequence), so the session needs another name.</summary>
|
||||
bool IsFolderTaken(string folder);
|
||||
}
|
||||
|
||||
/// <summary>Turns card photos into an <see cref="ImportPlan"/> (BRIEF §12.4, §13). Nothing is written.</summary>
|
||||
public static class ImportPlanner
|
||||
{
|
||||
public static async Task<ImportPlan> PlanAsync(string sourceRoot, string destinationRoot, IReadOnlyList<CardPhoto> photos,
|
||||
IReadOnlyList<ImportProblem> problems, ImportOptions options, IImportLookup lookup, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var folderTemplate = PathTemplate.Parse(options.FolderTemplate);
|
||||
var fileTemplate = PathTemplate.Parse(options.FileTemplate);
|
||||
var drafts = new List<(string Key, List<CardPhoto> Photos)>();
|
||||
foreach (var body in photos.GroupBy(BodyKey, StringComparer.Ordinal))
|
||||
{
|
||||
var ordered = body.OrderBy(p => p.CaptureTime).ThenBy(p => p.File.RelativePath, StringComparer.Ordinal).ToList();
|
||||
var ranges = SessionSplitter.Split(ordered.ConvertAll(p => p.CaptureTime), options.SessionGap, options.MinimumSessionFrames,
|
||||
i => options.SplitBefore.Contains(ordered[i].File.RelativePath) ? true : options.MergeAt.Contains(ordered[i].File.RelativePath) ? false : null);
|
||||
foreach (var range in ranges)
|
||||
{
|
||||
var members = ordered.GetRange(range.Start, range.Count);
|
||||
drafts.Add((SessionKey(body.Key, members[0].CaptureTime), members));
|
||||
}
|
||||
}
|
||||
drafts.Sort((a, b) => a.Photos[0].CaptureTime.CompareTo(b.Photos[0].CaptureTime) is var c and not 0 ? c : string.CompareOrdinal(a.Key, b.Key));
|
||||
|
||||
var claimedFolders = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var sessions = new List<PlannedSession>(drafts.Count);
|
||||
for (var s = 0; s < drafts.Count; s++)
|
||||
{
|
||||
var (key, members) = drafts[s];
|
||||
var first = members[0];
|
||||
options.Titles.TryGetValue(key, out var title);
|
||||
var context = Context(first, first.CaptureTime, s + 1, 1, title);
|
||||
var folder = await lookup.FindSessionFolderAsync(key, cancellationToken).ConfigureAwait(false)
|
||||
?? UniqueFolder(folderTemplate.ExpandFolder(context), claimedFolders, lookup);
|
||||
claimedFolders.Add(folder);
|
||||
|
||||
var names = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var items = new List<PlannedItem>(members.Count);
|
||||
for (var i = 0; i < members.Count; i++)
|
||||
{
|
||||
var photo = members[i];
|
||||
var name = UniqueName(fileTemplate.ExpandFile(Context(photo, first.CaptureTime, s + 1, i + 1, title)), names);
|
||||
items.Add(await PlanItemAsync(sourceRoot, photo, name, options, lookup, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
sessions.Add(new PlannedSession
|
||||
{
|
||||
Number = s + 1,
|
||||
SessionKey = key,
|
||||
CameraMake = first.Metadata.Make,
|
||||
CameraModel = first.Metadata.Model,
|
||||
CameraSerial = first.Metadata.BodySerial,
|
||||
Format = first.Metadata.Format,
|
||||
Start = first.CaptureTime,
|
||||
End = members[^1].CaptureTime,
|
||||
Folder = folder,
|
||||
Title = title,
|
||||
Items = items,
|
||||
});
|
||||
}
|
||||
return new ImportPlan
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
SourceRoot = Path.GetFullPath(sourceRoot),
|
||||
DestinationRoot = Path.GetFullPath(destinationRoot),
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
Sessions = sessions,
|
||||
Problems = problems,
|
||||
AvailableBytes = FreeSpace(destinationRoot),
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<PlannedItem> PlanItemAsync(string sourceRoot, CardPhoto photo, string name, ImportOptions options, IImportLookup lookup,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var item = new PlannedItem
|
||||
{
|
||||
SourcePath = photo.File.RelativePath,
|
||||
Size = photo.File.Size,
|
||||
ModifiedUtc = photo.File.ModifiedUtc,
|
||||
CapturedAt = photo.CaptureTime,
|
||||
FileName = name,
|
||||
};
|
||||
if (await lookup.WasImportedAsync(photo.File, cancellationToken).ConfigureAwait(false))
|
||||
return item with { Action = ImportAction.SkipImported };
|
||||
var candidates = await lookup.FindCandidatesAsync(photo.File.Size, photo.CaptureTime, cancellationToken).ConfigureAwait(false);
|
||||
if (candidates.Count == 0) return item;
|
||||
var sha = await ContentHash.Sha256Async(Path.Combine(sourceRoot, photo.File.RelativePath), null, cancellationToken).ConfigureAwait(false);
|
||||
var match = candidates.FirstOrDefault(c => sha.Equals(c.Sha256, StringComparison.OrdinalIgnoreCase));
|
||||
if (match is null) return item with { Sha256 = sha };
|
||||
return item with
|
||||
{
|
||||
Sha256 = sha,
|
||||
ExistingFrameId = match.FrameId,
|
||||
Action = options.Duplicates == DuplicatePolicy.Skip ? ImportAction.SkipInArchive : ImportAction.Copy,
|
||||
};
|
||||
}
|
||||
|
||||
private static NamingContext Context(CardPhoto photo, DateTime sessionStart, int session, int number, string? title) => new()
|
||||
{
|
||||
CapturedAt = photo.CaptureTime,
|
||||
SessionStart = sessionStart,
|
||||
Session = session,
|
||||
Number = number,
|
||||
NumberInSession = number,
|
||||
CameraMake = photo.Metadata.Make,
|
||||
CameraModel = photo.Metadata.Model,
|
||||
Lens = photo.Metadata.LensModel,
|
||||
Iso = photo.Metadata.Iso,
|
||||
ExposureTime = photo.Metadata.ExposureTime,
|
||||
FNumber = photo.Metadata.FNumber,
|
||||
OriginalName = Path.GetFileNameWithoutExtension(photo.File.RelativePath),
|
||||
Extension = Path.GetExtension(photo.File.RelativePath).TrimStart('.'),
|
||||
Title = title,
|
||||
};
|
||||
|
||||
/// <summary>Camera body and file format: RAW+JPEG shooting yields two sequences instead of doubled frames.</summary>
|
||||
private static string BodyKey(CardPhoto p) => $"{p.Metadata.Make}|{p.Metadata.Model}|{p.Metadata.BodySerial}|{p.Metadata.Format}";
|
||||
|
||||
private static string SessionKey(string bodyKey, DateTime start) =>
|
||||
"card-" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(bodyKey)))[..8] + "-"
|
||||
+ start.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
|
||||
|
||||
private static string UniqueFolder(string folder, HashSet<string> claimed, IImportLookup lookup)
|
||||
{
|
||||
if (folder.Length == 0) folder = "Import";
|
||||
var candidate = folder;
|
||||
for (var n = 2; claimed.Contains(candidate) || lookup.IsFolderTaken(candidate); n++)
|
||||
candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/// <summary>Adds _2, _3… before the extension when the template gives two photos the same name.</summary>
|
||||
internal static string UniqueName(string name, HashSet<string> taken)
|
||||
{
|
||||
var candidate = name;
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
var extension = Path.GetExtension(name);
|
||||
for (var n = 2; !taken.Add(candidate); n++) candidate = string.Create(CultureInfo.InvariantCulture, $"{stem}_{n}{extension}");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private static long? FreeSpace(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var full = Path.GetFullPath(path);
|
||||
while (!Directory.Exists(full) && Path.GetDirectoryName(full) is { } parent) full = parent;
|
||||
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
|
||||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>A run of consecutive frames forming one session: <c>[Start, Start + Count)</c>.</summary>
|
||||
public readonly record struct SessionRange(int Start, int Count)
|
||||
{
|
||||
public int End => Start + Count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits a time-ordered run of captures into shooting sessions (BRIEF §13): a pause longer than the gap starts a
|
||||
/// new session; a session with fewer than the minimum frames (test shots) merges into the previous one, or into the
|
||||
/// next when it comes first. Breaks the user forced are never merged away; breaks the user removed never split.
|
||||
/// </summary>
|
||||
public static class SessionSplitter
|
||||
{
|
||||
/// <param name="times">Capture times in ascending order.</param>
|
||||
/// <param name="userBreak">For a frame index: true = the user starts a session there, false = the user joined it
|
||||
/// to the previous frame, null = automatic.</param>
|
||||
public static IReadOnlyList<SessionRange> Split(IReadOnlyList<DateTime> times, TimeSpan gap, int minimumFrames, Func<int, bool?>? userBreak = null)
|
||||
{
|
||||
if (times.Count == 0) return [];
|
||||
var groups = new List<(int Start, int Count, bool Forced)>();
|
||||
var start = 0;
|
||||
var forced = false;
|
||||
for (var i = 1; i < times.Count; i++)
|
||||
{
|
||||
var decision = userBreak?.Invoke(i);
|
||||
if (decision == false || (decision is null && times[i] - times[i - 1] <= gap)) continue;
|
||||
groups.Add((start, i - start, forced));
|
||||
start = i;
|
||||
forced = decision == true;
|
||||
}
|
||||
groups.Add((start, times.Count - start, forced));
|
||||
|
||||
for (var i = 0; i < groups.Count && groups.Count > 1;)
|
||||
{
|
||||
var g = groups[i];
|
||||
if (g.Count >= minimumFrames)
|
||||
{
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (i > 0 && !g.Forced)
|
||||
{
|
||||
var previous = groups[i - 1];
|
||||
groups[i - 1] = (previous.Start, previous.Count + g.Count, previous.Forced);
|
||||
groups.RemoveAt(i);
|
||||
}
|
||||
else if (i == 0 && !groups[1].Forced)
|
||||
{
|
||||
groups[1] = (g.Start, g.Count + groups[1].Count, g.Forced);
|
||||
groups.RemoveAt(0);
|
||||
}
|
||||
else
|
||||
{
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return groups.ConvertAll(g => new SessionRange(g.Start, g.Count));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
using System.Security.Cryptography;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>The copy read back from the destination does not match what was read from the source.</summary>
|
||||
public sealed class CopyVerificationException(string path)
|
||||
: IOException(Loc.L("Archive.Error.CopyMismatch", Path.GetFileName(path)))
|
||||
{
|
||||
public string FilePath { get; } = path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies a file while hashing it, flushes the copy to disk, then re-reads it and compares hashes (BRIEF §12.4).
|
||||
/// A failed or interrupted copy never leaves a partial file behind.
|
||||
/// </summary>
|
||||
public static class VerifiedCopy
|
||||
{
|
||||
/// <summary>Copies <paramref name="source"/> to the new file <paramref name="destination"/>; returns the SHA-256.</summary>
|
||||
/// <param name="verify">Re-read the copy and compare (the default import setting).</param>
|
||||
/// <param name="bytesCopied">Receives the bytes of each chunk read from the source.</param>
|
||||
/// <exception cref="CopyVerificationException">The re-read copy differs.</exception>
|
||||
public static async Task<string> CopyAsync(string source, string destination, bool verify, Action<long>? bytesCopied = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(destination))!);
|
||||
var created = false;
|
||||
try
|
||||
{
|
||||
string sha;
|
||||
DateTime modified;
|
||||
await using (var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, ContentHash.BufferSize,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
await using (var output = new FileStream(destination, FileMode.CreateNew, FileAccess.Write, FileShare.None, ContentHash.BufferSize,
|
||||
FileOptions.Asynchronous))
|
||||
{
|
||||
created = true;
|
||||
modified = File.GetLastWriteTimeUtc(source);
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
var buffer = new byte[ContentHash.BufferSize];
|
||||
int read;
|
||||
while ((read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await output.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
|
||||
bytesCopied?.Invoke(read);
|
||||
}
|
||||
await output.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
output.Flush(flushToDisk: true);
|
||||
sha = Convert.ToHexStringLower(hash.GetHashAndReset());
|
||||
}
|
||||
File.SetLastWriteTimeUtc(destination, modified);
|
||||
if (verify && await ContentHash.Sha256Async(destination, null, cancellationToken).ConfigureAwait(false) != sha)
|
||||
throw new CopyVerificationException(destination);
|
||||
return sha;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Only our own partial or unverified copy; an existing file at the destination is never touched.
|
||||
if (created && File.Exists(destination)) File.Delete(destination);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.IO;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Frames processed by one thumbnail or proxy pass over a sequence.</summary>
|
||||
public sealed record ImagePassResult(int Built, int Reused, int Undecodable);
|
||||
|
||||
/// <summary>
|
||||
/// Thumbnails (<c>thumbs.pack</c>) and proxies (<c>proxies/0001.jpg</c>) of sequences (BRIEF §8.5). Both are
|
||||
/// regenerable and self-validating: a thumbnail carries its source fingerprint, a proxy its capture time, so after
|
||||
/// frames were renumbered the stale ones are rebuilt and the rest reused.
|
||||
/// </summary>
|
||||
public sealed class ArchiveImages
|
||||
{
|
||||
private const int CommitFrames = 200;
|
||||
private static readonly TimeSpan CommitInterval = TimeSpan.FromSeconds(10);
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
private readonly FrameImageRenderer _renderer;
|
||||
|
||||
internal ArchiveImages(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
_renderer = new FrameImageRenderer(writer.Options.Decoder);
|
||||
}
|
||||
|
||||
/// <param name="startFrame">Resume point inside the sequence (frames before it were done by an earlier run).</param>
|
||||
/// <param name="checkpoint">Called with the next frame index whenever the pack has been committed.</param>
|
||||
public async Task<ImagePassResult> BuildThumbnailsAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||||
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
int built = 0, reused = 0, undecodable = 0;
|
||||
// Thumbnail writers exclude each other only; ingest keeps committing meanwhile.
|
||||
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{sequence.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
||||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||||
using (var pack = ThumbPackWriter.Open(folder.ThumbsPackPath))
|
||||
{
|
||||
var cadence = new Cadence();
|
||||
for (var i = startFrame; i < manifest.Frames.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var frame = manifest.Frames[i];
|
||||
var fingerprint = ThumbPack.Fingerprint(frame.Sha256, frame.File, frame.Size);
|
||||
if (!force && pack.Entries.TryGetValue(frame.I, out var entry) && entry.SourceHash.AsSpan().SequenceEqual(fingerprint))
|
||||
{
|
||||
reused++;
|
||||
continue;
|
||||
}
|
||||
if (Render(() => _renderer.Thumbnail(folder, frame, cancellationToken)) is { } image)
|
||||
{
|
||||
pack.Append(frame.I, image.Width, image.Height, fingerprint, image.Jpeg);
|
||||
built++;
|
||||
}
|
||||
else
|
||||
{
|
||||
undecodable++;
|
||||
}
|
||||
if (cadence.Due())
|
||||
{
|
||||
pack.Commit();
|
||||
await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
pack.Commit();
|
||||
}
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ThumbsBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
|
||||
/// <inheritdoc cref="BuildThumbnailsAsync"/>
|
||||
public async Task<ImagePassResult> BuildProxiesAsync(Archive archive, Sequence sequence, bool force, int startFrame,
|
||||
Func<int, int, ValueTask> checkpoint, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var policy = ManifestStore.ReadArchive(paths.ManifestPath).ProxyPolicy;
|
||||
if (policy.LongEdge == ProxyPolicy.Disabled) return new ImagePassResult(0, 0, 0);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
Directory.CreateDirectory(folder.Proxies);
|
||||
int built = 0, reused = 0, undecodable = 0;
|
||||
var cadence = new Cadence();
|
||||
for (var i = startFrame; i < manifest.Frames.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var frame = manifest.Frames[i];
|
||||
var path = folder.Proxy(frame.I);
|
||||
if (!force && IsCurrent(path, frame))
|
||||
{
|
||||
reused++;
|
||||
continue;
|
||||
}
|
||||
if (Render(() => _renderer.Proxy(folder, frame, manifest.Camera, policy, cancellationToken)) is { } image)
|
||||
{
|
||||
AtomicFile.WriteAllBytes(path, image.Jpeg);
|
||||
built++;
|
||||
}
|
||||
else
|
||||
{
|
||||
undecodable++;
|
||||
}
|
||||
if (cadence.Due()) await checkpoint(i + 1, manifest.Frames.Count).ConfigureAwait(false);
|
||||
}
|
||||
await MarkAsync(archive, paths, sequence, SequenceStatus.ProxiesBuilt, undecodable == 0, cancellationToken).ConfigureAwait(false);
|
||||
return new ImagePassResult(built, reused, undecodable);
|
||||
}
|
||||
|
||||
/// <summary>A proxy is current when its Exif capture time is the frame's (frames renumbered since make it stale).</summary>
|
||||
private static bool IsCurrent(string proxy, SidecarFrame frame)
|
||||
{
|
||||
if (!File.Exists(proxy)) return false;
|
||||
try
|
||||
{
|
||||
return MetadataReader.Read(proxy).CapturedAt == frame.CapturedAt && frame.CapturedAt is not null;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A frame no decoder can read is skipped and counted; it never stops the pass.</summary>
|
||||
private static EncodedImage? Render(Func<EncodedImage?> render)
|
||||
{
|
||||
try
|
||||
{
|
||||
return render();
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or NotSupportedException)
|
||||
{
|
||||
Core.Diagnostics.Log.Warning("archive", "Frame image could not be rendered", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Commit/checkpoint rhythm: every <see cref="CommitFrames"/> rendered frames or <see cref="CommitInterval"/>.</summary>
|
||||
private sealed class Cadence
|
||||
{
|
||||
private readonly Stopwatch _clock = Stopwatch.StartNew();
|
||||
private int _frames;
|
||||
|
||||
public bool Due()
|
||||
{
|
||||
if (++_frames < CommitFrames && _clock.Elapsed < CommitInterval) return false;
|
||||
_frames = 0;
|
||||
_clock.Restart();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task MarkAsync(Archive archive, ArchivePaths paths, Sequence sequence, SequenceStatus flag, bool complete, CancellationToken cancellationToken)
|
||||
{
|
||||
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m =>
|
||||
{
|
||||
var status = complete ? m.Status | flag : m.Status & ~flag;
|
||||
return status == m.Status ? null : m with { Status = status, UpdatedAt = DateTimeOffset.UtcNow };
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpdateStatusAsync(sequence.Id, complete ? flag : SequenceStatus.None, complete ? SequenceStatus.None : flag, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
public enum ScanMode
|
||||
{
|
||||
/// <summary>Only folders whose sidecar or photo folder changed size or time since the last scan.</summary>
|
||||
Incremental,
|
||||
|
||||
/// <summary>Every folder: sidecars re-read and folders reconciled with their photos (manual rescan).</summary>
|
||||
Full,
|
||||
|
||||
/// <summary>The index rebuilt from sidecars alone; rows without a sidecar are dropped (BRIEF §12.5).</summary>
|
||||
Rebuild,
|
||||
}
|
||||
|
||||
public sealed record ScanProgress(int FoldersDone, int FoldersTotal, string Folder, bool Changed);
|
||||
|
||||
/// <param name="Failed">Folders that could not be indexed, with the reason; they are retried by the next scan.</param>
|
||||
public sealed record ScanSummary(int Folders, int Indexed, int Adopted, int Removed, IReadOnlyList<(string Folder, string Error)> Failed);
|
||||
|
||||
/// <summary>
|
||||
/// Keeps the database index in line with the archive on disk (BRIEF §7, §12.5): scans by size/time stamps, adopts
|
||||
/// folders without a sidecar, rebuilds from sidecars (using the index snapshot when it is fresh) and syncs one
|
||||
/// sequence after any service changed its sidecar. The sidecars are the truth; the index is derived.
|
||||
/// </summary>
|
||||
public sealed class ArchiveIndexer
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
internal ArchiveIndexer(TitanoDatabase db, SequenceWriter writer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
}
|
||||
|
||||
/// <param name="resumeAfter">Last folder completed by an interrupted run (job checkpoint), or null.</param>
|
||||
/// <param name="progress">Called after each folder; persist <see cref="ScanProgress.Folder"/> as the resume point.</param>
|
||||
/// <exception cref="ArchiveOfflineException">The archive is offline or read-only; the index is left as it is.</exception>
|
||||
public async Task<ScanSummary> ScanAsync(Archive archive, ScanMode mode, string? resumeAfter, Func<ScanProgress, ValueTask> progress,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
if (mode == ScanMode.Rebuild && resumeAfter is null) await _db.Archives.ClearFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var folders = ArchiveScanner.FindSequenceFolders(paths, cancellationToken);
|
||||
var states = await _db.Archives.GetFolderStatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var locations = new Locations(await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false));
|
||||
var snapshot = mode == ScanMode.Rebuild ? IndexSnapshot.TryLoad(paths.SnapshotPath) : null;
|
||||
using var snapshotWriter = mode != ScanMode.Incremental && resumeAfter is null ? new IndexSnapshot.Writer(paths.SnapshotPath) : null;
|
||||
|
||||
int indexed = 0, adopted = 0;
|
||||
var failed = new List<(string, string)>();
|
||||
for (var i = 0; i < folders.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var folder = folders[i];
|
||||
if (resumeAfter is not null && string.CompareOrdinal(folder.RelativeFolder, resumeAfter) <= 0) continue;
|
||||
states.TryGetValue(folder.RelativeFolder, out var state);
|
||||
var changed = mode != ScanMode.Incremental || state is null || state.Sidecar != folder.Sidecar || state.Photos != folder.Photos;
|
||||
if (changed)
|
||||
{
|
||||
try
|
||||
{
|
||||
var reconcile = mode == ScanMode.Full || (state is not null && state.Photos != folder.Photos);
|
||||
var bytes = await IndexFolderAsync(archive, paths, folder, reconcile, snapshot, locations, cancellationToken).ConfigureAwait(false);
|
||||
if (snapshotWriter is not null)
|
||||
{
|
||||
var sidecar = paths.Sequence(folder.RelativeFolder).ManifestPath;
|
||||
snapshotWriter.Add(folder.RelativeFolder, FileStamp.Of(sidecar)!.Value,
|
||||
bytes ?? await File.ReadAllBytesAsync(sidecar, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
indexed++;
|
||||
if (folder.Sidecar is null) adopted++;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"Indexing {folder.RelativeFolder} failed", ex);
|
||||
failed.Add((folder.RelativeFolder, ex.Message));
|
||||
}
|
||||
}
|
||||
await progress(new ScanProgress(i + 1, folders.Count, folder.RelativeFolder, changed)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var removed = mode == ScanMode.Rebuild ? await RemoveUnbackedAsync(archive, paths, folders, locations, cancellationToken).ConfigureAwait(false) : 0;
|
||||
if (failed.Count == 0) snapshotWriter?.Commit();
|
||||
return new ScanSummary(folders.Count, indexed, adopted, removed, failed);
|
||||
}
|
||||
|
||||
/// <summary>Writes a sequence's sidecar into the index (row, frames, session keys) and records the folder stamps.</summary>
|
||||
internal async Task SyncAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.SyncSequenceAsync(manifest.Id, SequenceMapping.ToFrames(manifest, sequence), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var key in manifest.SessionKeys)
|
||||
await tx.Sequences.MapSessionKeyAsync(archive.Id, key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Remembers the current stamps of a folder whose sidecar and index were just updated together, so the next
|
||||
/// incremental scan does not re-read what is already indexed.
|
||||
/// </summary>
|
||||
internal Task RecordStampsAsync(Archive archive, SequencePaths sequence, Guid sequenceId, CancellationToken cancellationToken) =>
|
||||
_db.Archives.SetFolderStateAsync(archive.Id, new FolderScanState(sequence.RelativeFolder, sequenceId,
|
||||
FileStamp.Of(sequence.ManifestPath), FileStamp.Of(ArchiveScanner.PhotoDirectory(sequence))), cancellationToken);
|
||||
|
||||
/// <summary>Indexes unindexed photos and flags missing files of one sequence (the orphans fix action).</summary>
|
||||
public async Task<SequenceManifest> ReconcileAsync(Archive archive, Sequence sequence, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var manifest = await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceAdopter.Reconcile(folder, m, cancellationToken),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await SyncAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>Indexes one folder; returns the sidecar bytes when they were read and not rewritten since (for the snapshot).</summary>
|
||||
private async Task<byte[]?> IndexFolderAsync(Archive archive, ArchivePaths paths, SequenceFolder folder, bool reconcile, IndexSnapshot? snapshot,
|
||||
Locations locations, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(folder.RelativeFolder);
|
||||
SequenceManifest manifest;
|
||||
byte[]? bytes = null;
|
||||
if (folder.Sidecar is not { } stamp)
|
||||
{
|
||||
manifest = await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
bytes = snapshot?.TryGet(folder.RelativeFolder, stamp) ?? await File.ReadAllBytesAsync(sequence.ManifestPath, cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
manifest = ManifestStore.ParseSequence(bytes, sequence.ManifestPath);
|
||||
}
|
||||
catch (InvalidDataException ex)
|
||||
{
|
||||
Log.Warning("archive", $"Damaged sidecar in {folder.RelativeFolder}; rebuilding it from the photos", ex);
|
||||
manifest = await ReplaceDamagedAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
||||
bytes = null;
|
||||
}
|
||||
}
|
||||
if (locations.IsCopy(paths, manifest.Id, folder.RelativeFolder))
|
||||
{
|
||||
manifest = Reidentify(manifest);
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
bytes = null;
|
||||
}
|
||||
if (reconcile || File.Exists(sequence.IngestJournalPath))
|
||||
{
|
||||
var before = manifest;
|
||||
manifest = await _writer.UpdateAsync(paths, manifest.Id, folder.RelativeFolder, m => SequenceAdopter.Reconcile(sequence, m, cancellationToken),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
if (!ReferenceEquals(before, manifest)) bytes = null;
|
||||
}
|
||||
await SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
locations.Set(manifest.Id, folder.RelativeFolder);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/// <summary>A sidecar for a folder that has none; ids of a sequence the index already knew at this folder are kept.</summary>
|
||||
private async Task<SequenceManifest> AdoptAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
|
||||
{
|
||||
var known = locations.AtFolder(sequence.RelativeFolder);
|
||||
var manifest = await WithKnownFrameIdsAsync(SequenceAdopter.Build(known ?? Guid.CreateVersion7(), sequence, _writer.Options.Owner, cancellationToken),
|
||||
sequence, cancellationToken).ConfigureAwait(false);
|
||||
Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
// Another process may adopt the same folder at the same moment: the first sidecar wins.
|
||||
return ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest) ? manifest : ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
}
|
||||
|
||||
/// <summary>Keeps the unreadable sidecar aside (never deleted) and builds a new one from the photos.</summary>
|
||||
private async Task<SequenceManifest> ReplaceDamagedAsync(SequencePaths sequence, Locations locations, CancellationToken cancellationToken)
|
||||
{
|
||||
File.Move(sequence.ManifestPath, $"{sequence.ManifestPath}.damaged-{DateTime.UtcNow:yyyyMMddHHmmss}");
|
||||
return await AdoptAsync(sequence, locations, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<SequenceManifest> WithKnownFrameIdsAsync(SequenceManifest manifest, SequencePaths sequence, CancellationToken cancellationToken)
|
||||
{
|
||||
var known = new Dictionary<string, Guid>(StringComparer.Ordinal);
|
||||
var prefix = sequence.ArchiveRelative(string.Empty);
|
||||
for (var after = -1; ;)
|
||||
{
|
||||
var page = await _db.Frames.ListBySequenceAsync(manifest.Id, after, 1000, cancellationToken).ConfigureAwait(false);
|
||||
foreach (var f in page)
|
||||
{
|
||||
if (f.RelativePath.StartsWith(prefix, StringComparison.Ordinal)) known[f.RelativePath[prefix.Length..]] = f.Id;
|
||||
}
|
||||
if (page.Count < 1000) break;
|
||||
after = page[^1].Index;
|
||||
}
|
||||
if (known.Count == 0) return manifest;
|
||||
return manifest with { Frames = manifest.Frames.Select(f => known.TryGetValue(f.File, out var id) ? f with { Id = id } : f).ToList() };
|
||||
}
|
||||
|
||||
/// <summary>A copied sequence folder carries the ids of its source; the copy gets ids of its own.</summary>
|
||||
private static SequenceManifest Reidentify(SequenceManifest manifest) => manifest with
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Frames = manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7() }).ToList(),
|
||||
SessionKeys = [],
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
/// <summary>Rebuild only: rows whose folder no longer holds a sidecar leave the index (their files are untouched).</summary>
|
||||
private async Task<int> RemoveUnbackedAsync(Archive archive, ArchivePaths paths, IReadOnlyList<SequenceFolder> folders, Locations locations,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var present = folders.Where(f => File.Exists(paths.Sequence(f.RelativeFolder).ManifestPath)).Select(f => f.RelativeFolder).ToHashSet(StringComparer.Ordinal);
|
||||
var removed = 0;
|
||||
foreach (var location in await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (present.Contains(location.Folder) && locations.AtFolder(location.Folder) == location.Id) continue;
|
||||
if (await _db.Sequences.DeleteAsync(location.Id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, location.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>Which folder each indexed sequence id lives in, updated as the scan goes.</summary>
|
||||
private sealed class Locations
|
||||
{
|
||||
private readonly Dictionary<Guid, string> _byId = [];
|
||||
private readonly Dictionary<string, Guid> _byFolder = new(StringComparer.Ordinal);
|
||||
|
||||
public Locations(IEnumerable<SequenceLocation> rows)
|
||||
{
|
||||
foreach (var row in rows) Set(row.Id, row.Folder);
|
||||
}
|
||||
|
||||
public Guid? AtFolder(string folder) => _byFolder.TryGetValue(folder, out var id) ? id : null;
|
||||
|
||||
/// <summary>The id is indexed at another folder whose sidecar still carries it: this folder is a copy.</summary>
|
||||
public bool IsCopy(ArchivePaths paths, Guid id, string folder)
|
||||
{
|
||||
if (!_byId.TryGetValue(id, out var other) || other == folder) return false;
|
||||
var otherSidecar = paths.Sequence(other).ManifestPath;
|
||||
try
|
||||
{
|
||||
return File.Exists(otherSidecar) && ManifestStore.ReadSequence(otherSidecar).Id == id;
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void Set(Guid id, string folder)
|
||||
{
|
||||
if (_byId.TryGetValue(id, out var previous) && _byFolder.GetValueOrDefault(previous) == id) _byFolder.Remove(previous);
|
||||
_byId[id] = folder;
|
||||
_byFolder[folder] = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>A verified staged file and how to name it.</summary>
|
||||
/// <param name="PlannedName">Name chosen by an import plan; null = the archive's file template.</param>
|
||||
internal sealed record StagedFrame(string StagedPath, string OriginalFileName, long Size, string Sha256, FrameMetadata Metadata,
|
||||
DateTime? ClientCapturedAt, string? PlannedName);
|
||||
|
||||
/// <summary>A sequence found or created by key (upload session, card session, or a new title).</summary>
|
||||
/// <param name="PreferredFolder">Folder of an import plan; null = the archive's folder template.</param>
|
||||
internal sealed record SequenceTarget(string Key, string? Title, string? PreferredFolder, string? CreatedBy);
|
||||
|
||||
/// <summary>
|
||||
/// Commits staged, verified files into sequences (BRIEF §4.3, §14.4): the file is renamed into the sequence folder
|
||||
/// under the naming template, the sidecar is rewritten atomically in capture order and the index follows. Commits
|
||||
/// to one sequence are serialised and batched (concurrent uploads share one sidecar write); a lease file keeps
|
||||
/// other processes out meanwhile, and a write-ahead journal repairs a batch interrupted by a crash. A file whose
|
||||
/// SHA-256 the sequence already holds is reported as a duplicate and left in staging.
|
||||
/// </summary>
|
||||
public sealed class ArchiveIngest : IArchiveIngest
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
private readonly ConcurrentDictionary<Guid, CommitQueue> _queues = new();
|
||||
private readonly SemaphoreSlim _creation = new(1, 1);
|
||||
|
||||
internal ArchiveIngest(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
public async Task<IngestResult> CommitAsync(IngestRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
if ((request.SequenceId is null ? 0 : 1) + (request.NewSequenceTitle is null ? 0 : 1) + (request.SessionKey is null ? 0 : 1) != 1)
|
||||
throw new ArgumentException("Exactly one of SequenceId, NewSequenceTitle or SessionKey selects the target.", nameof(request));
|
||||
var archive = await _db.Archives.GetAsync(request.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown archive {request.ArchiveId}.", nameof(request));
|
||||
var staged = new FileInfo(request.StagedPath);
|
||||
if (!staged.Exists || staged.Length != request.Size)
|
||||
throw new FileNotFoundException("The staged file is missing or has another size.", request.StagedPath);
|
||||
var frame = new StagedFrame(request.StagedPath, request.OriginalFileName, request.Size, request.Sha256,
|
||||
MetadataReader.Read(request.StagedPath, cancellationToken), request.ClientCapturedAt, null);
|
||||
|
||||
Guid sequenceId;
|
||||
if (request.SequenceId is { } id)
|
||||
{
|
||||
var sequence = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
if (sequence?.ArchiveId != archive.Id) throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(request));
|
||||
sequenceId = id;
|
||||
}
|
||||
else
|
||||
{
|
||||
// A title names one sequence per user: parallel uploads for "new sequence X" all land in the same one.
|
||||
var key = request.SessionKey ?? $"title:{request.UserId:N}:{request.NewSequenceTitle!.Trim().ToLowerInvariant()}";
|
||||
sequenceId = await FindOrCreateAsync(archive, new SequenceTarget(key, request.NewSequenceTitle?.Trim(), null, request.UserId?.ToString()),
|
||||
frame, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return (await EnqueueAsync(archive, sequenceId, [frame], cancellationToken).ConfigureAwait(false))[0];
|
||||
}
|
||||
|
||||
/// <summary>Commits files of one import session into the sequence of its key (created on first use).</summary>
|
||||
internal async Task<IReadOnlyList<IngestResult>> CommitSessionAsync(Archive archive, SequenceTarget target, IReadOnlyList<StagedFrame> frames,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (frames.Count == 0) return [];
|
||||
var sequenceId = await FindOrCreateAsync(archive, target, frames[0], cancellationToken).ConfigureAwait(false);
|
||||
return await EnqueueAsync(archive, sequenceId, frames, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task<Guid> FindOrCreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
|
||||
{
|
||||
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } found) return found;
|
||||
await _creation.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (await _db.Sequences.FindBySessionKeyAsync(archive.Id, target.Key, cancellationToken).ConfigureAwait(false) is { } again) return again;
|
||||
return await CreateAsync(archive, target, first, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_creation.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Guid> CreateAsync(Archive archive, SequenceTarget target, StagedFrame first, CancellationToken cancellationToken)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var m = first.Metadata;
|
||||
var manifest = new SequenceManifest
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Title = target.Title,
|
||||
Camera = new SidecarCamera { Make = m.Make, Model = m.Model, Serial = m.BodySerial },
|
||||
Lens = m.LensModel,
|
||||
SessionKeys = [target.Key],
|
||||
CreatedBy = target.CreatedBy,
|
||||
Created = now,
|
||||
UpdatedAt = now,
|
||||
};
|
||||
var captured = CaptureTime(first);
|
||||
var wanted = target.PreferredFolder ?? PathTemplate.Parse(archive.FolderTemplate).ExpandFolder(Naming(first, captured, captured, 1, target.Title));
|
||||
if (wanted.Length == 0) wanted = "Sequence";
|
||||
var taken = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false))
|
||||
.Select(l => l.Folder).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var sequence = ClaimFolder(paths, wanted, taken, manifest);
|
||||
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, null), cancellationToken).ConfigureAwait(false);
|
||||
var winner = await tx.Sequences.MapSessionKeyAsync(archive.Id, target.Key, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
if (winner != manifest.Id)
|
||||
{
|
||||
// Another process bound the key first: its sequence wins, ours was never used.
|
||||
await tx.RollbackAsync(cancellationToken).ConfigureAwait(false);
|
||||
ReleaseFolder(sequence);
|
||||
return winner;
|
||||
}
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
return manifest.Id;
|
||||
}
|
||||
|
||||
/// <summary>The first free variant of <paramref name="wanted"/>, claimed by creating its sidecar (first writer wins).</summary>
|
||||
private static SequencePaths ClaimFolder(ArchivePaths paths, string wanted, HashSet<string> taken, SequenceManifest manifest)
|
||||
{
|
||||
for (var n = 1; ; n++)
|
||||
{
|
||||
var folder = n == 1 ? wanted : string.Create(CultureInfo.InvariantCulture, $"{wanted} ({n})");
|
||||
if (taken.Contains(folder)) continue;
|
||||
var sequence = paths.Sequence(folder);
|
||||
if (Directory.Exists(sequence.Folder) && Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) continue;
|
||||
Directory.CreateDirectory(sequence.MetaDirectory);
|
||||
if (ManifestStore.TryCreateSequence(sequence.ManifestPath, manifest)) return sequence;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Removes the empty sequence this process created and lost to a concurrent creator.</summary>
|
||||
private static void ReleaseFolder(SequencePaths sequence)
|
||||
{
|
||||
File.Delete(sequence.ManifestPath);
|
||||
if (!Directory.EnumerateFileSystemEntries(sequence.MetaDirectory).Any()) Directory.Delete(sequence.MetaDirectory);
|
||||
if (!Directory.EnumerateFileSystemEntries(sequence.Folder).Any()) Directory.Delete(sequence.Folder);
|
||||
}
|
||||
|
||||
/// <summary>Group commit: whoever gets the sequence's gate commits everything queued for it.</summary>
|
||||
private async Task<IngestResult[]> EnqueueAsync(Archive archive, Guid sequenceId, IReadOnlyList<StagedFrame> frames, CancellationToken cancellationToken)
|
||||
{
|
||||
var queue = _queues.GetOrAdd(sequenceId, _ => new CommitQueue());
|
||||
var pending = frames.Select(f => new Pending(f)).ToList();
|
||||
foreach (var p in pending) queue.Items.Enqueue(p);
|
||||
await queue.Gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (pending.Any(p => !p.Completion.Task.IsCompleted))
|
||||
{
|
||||
var batch = new List<Pending>();
|
||||
while (queue.Items.TryDequeue(out var p)) batch.Add(p);
|
||||
try
|
||||
{
|
||||
await CommitBatchAsync(archive, sequenceId, batch, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
foreach (var p in batch) p.Completion.TrySetException(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
queue.Gate.Release();
|
||||
}
|
||||
return await Task.WhenAll(pending.Select(p => p.Completion.Task)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task CommitBatchAsync(Archive archive, Guid sequenceId, List<Pending> batch, CancellationToken cancellationToken)
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Sequence {sequenceId} is not indexed.");
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var sequence = paths.Sequence(row.Folder);
|
||||
await using (await _writer.LockAsync(paths, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, sequenceId);
|
||||
var before = manifest.Frames.Count;
|
||||
var known = new Dictionary<string, SidecarFrame>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var f in manifest.Frames)
|
||||
{
|
||||
if (f.Sha256 is not null) known.TryAdd(f.Sha256, f);
|
||||
}
|
||||
|
||||
var accepted = new List<(Pending Pending, SidecarFrame Frame)>();
|
||||
var duplicates = new List<(Pending Pending, Guid FrameId)>();
|
||||
foreach (var p in batch)
|
||||
{
|
||||
if (known.TryGetValue(p.Frame.Sha256, out var existing))
|
||||
{
|
||||
duplicates.Add((p, existing.Id));
|
||||
continue;
|
||||
}
|
||||
var frame = SequenceFrames.FromMetadata(Guid.CreateVersion7(), p.Frame.OriginalFileName, p.Frame.OriginalFileName, p.Frame.Size,
|
||||
p.Frame.Sha256, p.Frame.Metadata, p.Frame.ClientCapturedAt);
|
||||
known[p.Frame.Sha256] = frame;
|
||||
accepted.Add((p, frame));
|
||||
}
|
||||
|
||||
var named = Name(archive, sequence, manifest, accepted);
|
||||
IngestJournal.Write(sequence, new IngestJournal(named.Values.ToList()));
|
||||
var moved = new List<SidecarFrame>();
|
||||
foreach (var (p, frame) in accepted)
|
||||
{
|
||||
var final = named[frame.Id];
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(sequence.File(final.File))!);
|
||||
File.Move(p.Frame.StagedPath, sequence.File(final.File), overwrite: false);
|
||||
moved.Add(final);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
p.Completion.TrySetException(ex);
|
||||
}
|
||||
}
|
||||
var updated = SequenceFrames.Insert(manifest, moved);
|
||||
if (moved.Count > 0) ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
IngestJournal.Delete(sequence);
|
||||
|
||||
var byId = updated.Frames.ToDictionary(f => f.Id);
|
||||
if (moved.Count > 0) await IndexAsync(archive, sequence, updated, before, moved.ConvertAll(m => byId[m.Id]), cancellationToken).ConfigureAwait(false);
|
||||
foreach (var (p, frame) in accepted.Where(a => byId.ContainsKey(a.Frame.Id)))
|
||||
p.Completion.TrySetResult(Result(sequenceId, sequence, byId[frame.Id], false));
|
||||
foreach (var (p, frameId) in duplicates)
|
||||
{
|
||||
if (byId.TryGetValue(frameId, out var original)) p.Completion.TrySetResult(Result(sequenceId, sequence, original, true));
|
||||
else p.Completion.TrySetException(new IOException(Loc.L("Archive.Error.DuplicateLost", p.Frame.OriginalFileName)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Final file names, in capture order: the plan's name or the template with <c>{n}</c> = index + 1, made unique.</summary>
|
||||
private static Dictionary<Guid, SidecarFrame> Name(Archive archive, SequencePaths sequence, SequenceManifest manifest,
|
||||
List<(Pending Pending, SidecarFrame Frame)> accepted)
|
||||
{
|
||||
var merged = SequenceFrames.Insert(manifest, accepted.ConvertAll(a => a.Frame));
|
||||
var incoming = accepted.ToDictionary(a => a.Frame.Id);
|
||||
// Photos adopted in place stay in place; everything else goes to originals/.
|
||||
var directory = Directory.Exists(sequence.Originals) || manifest.Frames.Count == 0 || manifest.Frames.Any(f => f.File.Contains('/'))
|
||||
? ArchiveLayout.OriginalsFolder + "/"
|
||||
: string.Empty;
|
||||
var taken = new HashSet<string>(manifest.Frames.Select(f => f.File), StringComparer.OrdinalIgnoreCase);
|
||||
var onDisk = Path.Combine(sequence.Folder, directory);
|
||||
if (Directory.Exists(onDisk))
|
||||
{
|
||||
foreach (var file in Directory.EnumerateFiles(onDisk)) taken.Add(directory + Path.GetFileName(file));
|
||||
}
|
||||
var template = PathTemplate.Parse(archive.FileTemplate);
|
||||
var start = merged.Frames.Count > 0 ? merged.Frames[0].CapturedAt : null;
|
||||
var named = new Dictionary<Guid, SidecarFrame>();
|
||||
foreach (var frame in merged.Frames)
|
||||
{
|
||||
if (!incoming.TryGetValue(frame.Id, out var entry)) continue;
|
||||
var staged = entry.Pending.Frame;
|
||||
var captured = frame.CapturedAt ?? DateTime.Now;
|
||||
var name = staged.PlannedName ?? template.ExpandFile(Naming(staged, captured, start ?? captured, frame.I + 1, manifest.Title));
|
||||
var file = directory + name;
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
for (var n = 2; !taken.Add(file); n++) file = string.Create(CultureInfo.InvariantCulture, $"{directory}{stem}_{n}{Path.GetExtension(name)}");
|
||||
named[frame.Id] = entry.Frame with { File = file };
|
||||
}
|
||||
return named;
|
||||
}
|
||||
|
||||
/// <summary>Appends the new rows when the index was in step with the sidecar; otherwise resyncs the whole sequence.</summary>
|
||||
private async Task IndexAsync(Archive archive, SequencePaths sequence, SequenceManifest manifest, int before, List<SidecarFrame> appended,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var inStep = appended.All(f => f.I >= before)
|
||||
&& await _db.Frames.CountBySequenceAsync(manifest.Id, cancellationToken).ConfigureAwait(false) == before;
|
||||
if (!inStep)
|
||||
{
|
||||
await _indexer.SyncAsync(archive, sequence, manifest, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await tx.Frames.BulkInsertAsync(appended.Select(f => SequenceMapping.ToFrame(f, manifest.Id, sequence)), cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, manifest.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static IngestResult Result(Guid sequenceId, SequencePaths sequence, SidecarFrame frame, bool duplicate) =>
|
||||
new(sequenceId, frame.Id, frame.I, sequence.ArchiveRelative(frame.File), duplicate);
|
||||
|
||||
private static DateTime CaptureTime(StagedFrame frame) =>
|
||||
frame.Metadata.TimestampSource is TimestampSource.FileSystem or TimestampSource.Unknown && frame.ClientCapturedAt is { } client
|
||||
? client
|
||||
: frame.Metadata.CapturedAt ?? frame.ClientCapturedAt ?? DateTime.Now;
|
||||
|
||||
private static NamingContext Naming(StagedFrame frame, DateTime captured, DateTime sessionStart, int number, string? title) => new()
|
||||
{
|
||||
CapturedAt = captured,
|
||||
SessionStart = sessionStart,
|
||||
Number = number,
|
||||
NumberInSession = number,
|
||||
CameraMake = frame.Metadata.Make,
|
||||
CameraModel = frame.Metadata.Model,
|
||||
Lens = frame.Metadata.LensModel,
|
||||
Iso = frame.Metadata.Iso,
|
||||
ExposureTime = frame.Metadata.ExposureTime,
|
||||
FNumber = frame.Metadata.FNumber,
|
||||
OriginalName = Path.GetFileNameWithoutExtension(frame.OriginalFileName),
|
||||
Extension = Path.GetExtension(frame.OriginalFileName).TrimStart('.'),
|
||||
Title = title,
|
||||
};
|
||||
|
||||
private sealed class CommitQueue
|
||||
{
|
||||
public SemaphoreSlim Gate { get; } = new(1, 1);
|
||||
public ConcurrentQueue<Pending> Items { get; } = new();
|
||||
}
|
||||
|
||||
private sealed class Pending(StagedFrame frame)
|
||||
{
|
||||
public StagedFrame Frame { get; } = frame;
|
||||
public TaskCompletionSource<IngestResult> Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original.</summary>
|
||||
public sealed record DuplicateGroup(string Sha256, IReadOnlyList<DuplicateFrame> Frames);
|
||||
|
||||
/// <summary>Index and disk out of step (BRIEF §12.5 Orphans).</summary>
|
||||
/// <param name="Sequences">Per indexed sequence: photos the sidecar does not list and frames whose file is gone.</param>
|
||||
/// <param name="MissingSequences">Index rows whose folder holds no sidecar any more.</param>
|
||||
/// <param name="UnindexedFolders">Folders with photos or a sidecar that the index does not know yet (a scan adopts them).</param>
|
||||
public sealed record OrphanReport(IReadOnlyList<SequenceOrphans> Sequences, IReadOnlyList<SequenceLocation> MissingSequences, IReadOnlyList<string> UnindexedFolders)
|
||||
{
|
||||
public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count);
|
||||
public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count);
|
||||
}
|
||||
|
||||
/// <summary>What a maintenance run removed or changed (or would, in a dry run).</summary>
|
||||
public sealed record MaintenanceReport(
|
||||
bool DryRun,
|
||||
IReadOnlyList<string> StaleLeases,
|
||||
IReadOnlyList<string> TemporaryFiles,
|
||||
IReadOnlyList<string> CompactedPacks,
|
||||
int DuplicateGroups,
|
||||
int DuplicateFlagChanges,
|
||||
int UnindexedFiles,
|
||||
int MissingFiles);
|
||||
|
||||
/// <summary>Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched.</summary>
|
||||
public sealed class ArchiveMaintenance
|
||||
{
|
||||
/// <summary>Staged copies older than this belong to no running import.</summary>
|
||||
private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>Temporary files of atomic writes older than this were left by a crash.</summary>
|
||||
private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>Packs with more reclaimable bytes than this share are compacted.</summary>
|
||||
private const double CompactionRatio = 0.3;
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
|
||||
internal ArchiveMaintenance(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
(await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false))
|
||||
.GroupBy(f => f.Sha256, StringComparer.Ordinal)
|
||||
.Select(g => new DuplicateGroup(g.Key, g.ToList()))
|
||||
.ToList();
|
||||
|
||||
public async Task<OrphanReport> FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var sequences = new List<SequenceOrphans>();
|
||||
var missing = new List<SequenceLocation>();
|
||||
foreach (var location in locations)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var sequence = paths.Sequence(location.Folder);
|
||||
if (!File.Exists(sequence.ManifestPath))
|
||||
{
|
||||
missing.Add(location);
|
||||
continue;
|
||||
}
|
||||
var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath));
|
||||
if (!orphans.IsEmpty) sequences.Add(orphans);
|
||||
}
|
||||
var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal);
|
||||
var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList();
|
||||
return new OrphanReport(sequences, missing, unindexed);
|
||||
}
|
||||
|
||||
/// <summary>Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept.</summary>
|
||||
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
|
||||
var removed = 0;
|
||||
foreach (var id in sequenceIds)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
|
||||
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath)) continue;
|
||||
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
|
||||
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/// <summary>Number of maintenance steps (the job checkpoints after each).</summary>
|
||||
internal const int StepCount = 4;
|
||||
|
||||
/// <summary>Runs the maintenance steps from <paramref name="firstStep"/>: stale leases, leftover temporary files,
|
||||
/// thumbnail pack compaction, duplicate flags. A dry run only reports.</summary>
|
||||
public async Task<MaintenanceReport> RunAsync(Archive archive, bool dryRun, int firstStep, Func<int, ValueTask> stepDone,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
if (!dryRun) ArchiveVolume.EnsureWritable(paths);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
IReadOnlyList<string> leases = [], temps = [], packs = [];
|
||||
int groups = 0, flagChanges = 0;
|
||||
for (var step = firstStep; step < StepCount; step++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
switch (step)
|
||||
{
|
||||
case 0:
|
||||
leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
|
||||
break;
|
||||
case 1:
|
||||
temps = PurgeTemporaryFiles(paths, locations, dryRun);
|
||||
break;
|
||||
case 2:
|
||||
packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
|
||||
break;
|
||||
case 3:
|
||||
(groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false);
|
||||
break;
|
||||
}
|
||||
await stepDone(step + 1).ConfigureAwait(false);
|
||||
}
|
||||
var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles);
|
||||
}
|
||||
|
||||
/// <summary>Staging copies and atomic-write temporaries left behind by crashes (never originals).</summary>
|
||||
private static List<string> PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
var found = new List<string>();
|
||||
void Collect(string directory, string pattern, TimeSpan age, bool recursive)
|
||||
{
|
||||
if (!Directory.Exists(directory)) return;
|
||||
foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
|
||||
}
|
||||
}
|
||||
Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
|
||||
Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
|
||||
foreach (var location in locations)
|
||||
{
|
||||
var sequence = paths.Sequence(location.Folder);
|
||||
Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false);
|
||||
Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false);
|
||||
}
|
||||
if (!dryRun)
|
||||
{
|
||||
foreach (var file in found)
|
||||
{
|
||||
try
|
||||
{
|
||||
File.Delete(file);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
Log.Warning("archive", $"Could not remove {file}", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
return found.ConvertAll(paths.ToRelative);
|
||||
}
|
||||
|
||||
private async Task<List<string>> CompactPacksAsync(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun, CancellationToken cancellationToken)
|
||||
{
|
||||
var compacted = new List<string>();
|
||||
foreach (var location in locations)
|
||||
{
|
||||
var sequence = paths.Sequence(location.Folder);
|
||||
long garbage, length;
|
||||
using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath))
|
||||
{
|
||||
garbage = reader.GarbageBytes;
|
||||
length = reader.Length;
|
||||
}
|
||||
if (length == 0 || garbage < length * CompactionRatio) continue;
|
||||
compacted.Add(location.Folder);
|
||||
if (dryRun) continue;
|
||||
await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
|
||||
ThumbPackWriter.Compact(sequence.ThumbsPackPath);
|
||||
}
|
||||
return compacted;
|
||||
}
|
||||
|
||||
/// <summary>Sets <see cref="FrameFlags.Duplicate"/> on every copy after the first of each hash and clears it elsewhere.</summary>
|
||||
private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken)
|
||||
{
|
||||
var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId);
|
||||
var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false))
|
||||
.ToDictionary(f => f.FrameId, f => f.SequenceId);
|
||||
var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList();
|
||||
var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList();
|
||||
if (dryRun) return (groups.Count, set.Count + clear.Count);
|
||||
|
||||
foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value))
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
|
||||
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
|
||||
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
|
||||
await _writer.UpdateAsync(paths, sequence.Id, sequence.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
|
||||
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
|
||||
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
|
||||
if (setIds.Count > 0) await _db.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
|
||||
if (clearIds.Count > 0) await _db.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(sequence.Folder), sequence.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return (groups.Count, set.Count + clear.Count);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// What the desktop client and the server call to work with archives (BRIEF §7, §12.4, §12.5, §14.4): open, create
|
||||
/// or adopt an archive, plan and apply mass renames, plan imports and card erases, query duplicates and orphans, and
|
||||
/// the job handlers for the long operations (enqueue them with <see cref="ArchiveJobs"/>).
|
||||
/// </summary>
|
||||
public sealed class ArchiveService
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
|
||||
public ArchiveService(TitanoDatabase db, ArchiveServiceOptions? options = null)
|
||||
{
|
||||
_db = db;
|
||||
_writer = new SequenceWriter(options ?? new ArchiveServiceOptions());
|
||||
Indexer = new ArchiveIndexer(db, _writer);
|
||||
Ingest = new ArchiveIngest(db, _writer, Indexer);
|
||||
Verifier = new ArchiveVerifier(db, _writer, Indexer);
|
||||
Maintenance = new ArchiveMaintenance(db, _writer, Indexer);
|
||||
Renamer = new MassRenamer(db, _writer, Indexer);
|
||||
Images = new ArchiveImages(db, _writer, Indexer);
|
||||
Imports = new ImportService(db, Ingest);
|
||||
}
|
||||
|
||||
public ArchiveIndexer Indexer { get; }
|
||||
|
||||
/// <summary>Commits staged files into sequences (server uploads, client card imports).</summary>
|
||||
public ArchiveIngest Ingest { get; }
|
||||
|
||||
public ArchiveVerifier Verifier { get; }
|
||||
public ArchiveMaintenance Maintenance { get; }
|
||||
public MassRenamer Renamer { get; }
|
||||
public ArchiveImages Images { get; }
|
||||
public ImportService Imports { get; }
|
||||
|
||||
/// <summary>Handlers for scan, rebuild, verify, import, thumbs, proxies and maintenance jobs; register them on the <see cref="JobEngine"/>.</summary>
|
||||
public IReadOnlyList<IJobHandler> CreateJobHandlers() =>
|
||||
[
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Scan),
|
||||
new ScanJobHandler(_db, Indexer, JobKind.Rebuild),
|
||||
new VerifyJobHandler(_db, Verifier),
|
||||
new ImportJobHandler(Imports),
|
||||
new ImageJobHandler(_db, Images, JobKind.Thumbs),
|
||||
new ImageJobHandler(_db, Images, JobKind.Proxies),
|
||||
new MaintenanceJobHandler(_db, Maintenance),
|
||||
];
|
||||
|
||||
/// <summary>Makes <paramref name="root"/> a new, empty archive and indexes it.</summary>
|
||||
/// <exception cref="InvalidOperationException">The folder already is an archive (open it instead).</exception>
|
||||
public async Task<Archive> CreateAsync(string root, string name, LayoutTemplates? templates = null, ProxyPolicy? proxies = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
if (File.Exists(paths.ManifestPath)) throw new InvalidOperationException($"{paths.Root} already is a Titano archive.");
|
||||
Directory.CreateDirectory(paths.MetaDirectory);
|
||||
var manifest = new ArchiveManifest
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Name = name,
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
LayoutTemplates = templates ?? new LayoutTemplates(),
|
||||
ProxyPolicy = proxies ?? new ProxyPolicy(),
|
||||
};
|
||||
ManifestStore.WriteArchive(paths.ManifestPath, manifest);
|
||||
return await RegisterAsync(paths, manifest, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers an existing archive (or relinks a moved one: same id, new root) and completes renames a crash
|
||||
/// interrupted. An offline archive is registered as it was last seen.
|
||||
/// </summary>
|
||||
public async Task<Archive> OpenAsync(string root, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
var archive = await RegisterAsync(paths, ManifestStore.ReadArchive(paths.ManifestPath), cancellationToken).ConfigureAwait(false);
|
||||
if (ArchiveVolume.Probe(paths) != ArchiveAvailability.Online) return archive;
|
||||
try
|
||||
{
|
||||
await Renamer.ResumePendingAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or InvalidDataException)
|
||||
{
|
||||
// The journal stays; the next open tries again. The archive itself is usable meanwhile.
|
||||
Log.Warning("archive", $"Resuming an interrupted rename in {paths.Root} failed", ex);
|
||||
}
|
||||
return archive;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens a folder of photos as an archive in place (Welcome "adopt folder"): nothing is moved or renamed; enqueue
|
||||
/// <see cref="ArchiveJobs.Scan"/> with a full pass to write sidecars and index it.
|
||||
/// </summary>
|
||||
public Task<Archive> AdoptAsync(string root, string? name = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(root);
|
||||
return File.Exists(paths.ManifestPath)
|
||||
? OpenAsync(root, cancellationToken)
|
||||
: CreateAsync(root, name ?? Path.GetFileName(paths.Root), null, null, cancellationToken);
|
||||
}
|
||||
|
||||
public static ArchiveAvailability GetAvailability(Archive archive) => ArchiveVolume.Probe(new ArchivePaths(archive.RootPath));
|
||||
|
||||
/// <summary>Changes title, tags, rating and notes in the sidecar and the index together (settings version + 1).</summary>
|
||||
public async Task<Sequence> SetSequenceInfoAsync(Guid sequenceId, string? title, IReadOnlyList<string> tags, int rating, string? notes,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var manifest = await _writer.UpdateAsync(paths, sequenceId, sequence.Folder, m => m with
|
||||
{
|
||||
Title = string.IsNullOrWhiteSpace(title) ? null : title.Trim(),
|
||||
Tags = tags,
|
||||
Rating = rating,
|
||||
Notes = notes,
|
||||
Versions = m.Versions with { Settings = System.Math.Max(m.Versions.Settings, sequence.SettingsVersion) + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var folder = paths.Sequence(sequence.Folder);
|
||||
var updated = SequenceMapping.ToSequence(manifest, archive.Id, folder, sequence);
|
||||
await _db.Sequences.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
await Indexer.RecordStampsAsync(archive, folder, sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
return updated;
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
Maintenance.FindDuplicatesAsync(archiveId, cancellationToken);
|
||||
|
||||
public async Task<OrphanReport> FindOrphansAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
await Maintenance.FindOrphansAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Orphans fix: index the unlisted photos of a sequence and flag its missing files.</summary>
|
||||
public async Task ReconcileAsync(Guid sequenceId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var (archive, sequence) = await LoadSequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await Indexer.ReconcileAsync(archive, sequence, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Orphans fix: drop index rows whose sidecar is gone.</summary>
|
||||
public async Task<int> RemoveMissingSequencesAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default) =>
|
||||
await Maintenance.RemoveMissingSequencesAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <summary>Dry run of a mass rename.</summary>
|
||||
public async Task<RenamePlan> PlanRenameAsync(Guid archiveId, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Renamer.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sequenceIds, folderTemplate, titleTemplate,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public async Task ApplyRenameAsync(RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
await Renamer.ApplyAsync(await LoadArchiveAsync(plan.ArchiveId, cancellationToken).ConfigureAwait(false), plan, progress, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <summary>Plans a card or folder import into an archive (nothing is copied).</summary>
|
||||
public async Task<ImportPlan> PlanImportAsync(Guid archiveId, string sourceRoot, ImportOptions options, Guid? cardId = null, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Imports.PlanAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), sourceRoot, options, cardId, progress,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Dry run of "delete imported" for a card; pass the plan to <see cref="CardEraser.Execute"/> after the user confirms.</summary>
|
||||
public async Task<ErasePlan> PlanCardEraseAsync(Guid archiveId, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
await Imports.PlanEraseAsync(await LoadArchiveAsync(archiveId, cancellationToken).ConfigureAwait(false), cardId, sourceRoot, progress,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
private async Task<Archive> RegisterAsync(ArchivePaths paths, ArchiveManifest manifest, CancellationToken cancellationToken)
|
||||
{
|
||||
var archive = new Archive
|
||||
{
|
||||
Id = manifest.Id,
|
||||
Name = manifest.Name,
|
||||
RootPath = paths.Root,
|
||||
FolderTemplate = manifest.LayoutTemplates.Folder,
|
||||
FileTemplate = manifest.LayoutTemplates.File,
|
||||
SchemaVersion = manifest.SchemaVersion,
|
||||
Created = manifest.Created,
|
||||
ProxyLongEdge = manifest.ProxyPolicy.LongEdge,
|
||||
};
|
||||
await _db.Archives.UpsertAsync(archive, cancellationToken).ConfigureAwait(false);
|
||||
return archive;
|
||||
}
|
||||
|
||||
private async Task<Archive> LoadArchiveAsync(Guid archiveId, CancellationToken cancellationToken) =>
|
||||
await _db.Archives.GetAsync(archiveId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown archive {archiveId}.", nameof(archiveId));
|
||||
|
||||
private async Task<(Archive, Sequence)> LoadSequenceAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new ArgumentException($"Unknown sequence {sequenceId}.", nameof(sequenceId));
|
||||
return (await LoadArchiveAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false), sequence);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Builds the archive jobs; enqueue with <see cref="EnqueueAsync"/>.</summary>
|
||||
public static class ArchiveJobs
|
||||
{
|
||||
public static ArchiveJobRequest Scan(Archive archive, bool full = false) =>
|
||||
new(JobKind.Scan, [archive.Id], archive.Name, Json(new ScanParameters(full), ArchiveJobJson.Default.ScanParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Rebuild(Archive archive) => new(JobKind.Rebuild, [archive.Id], archive.Name, null, JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Verify(Archive archive, IReadOnlyList<Guid>? sequenceIds = null) =>
|
||||
new(JobKind.Verify, [archive.Id], archive.Name, Json(new VerifyParameters(sequenceIds), ArchiveJobJson.Default.VerifyParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Import(Archive archive, ImportPlan plan, Guid? cardId, bool verify = true, Guid? userId = null) =>
|
||||
new(JobKind.Import, [archive.Id], archive.Name,
|
||||
Json(new ImportJobParameters(archive.Id, plan, cardId, verify, userId), ArchiveJobJson.Default.ImportJobParameters), JobPriority.User);
|
||||
|
||||
public static ArchiveJobRequest Thumbnails(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
|
||||
new(JobKind.Thumbs, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
|
||||
|
||||
public static ArchiveJobRequest Proxies(IReadOnlyList<Guid> sequenceIds, string? label = null, bool force = false) =>
|
||||
new(JobKind.Proxies, sequenceIds, label, Json(new ImageJobParameters(force), ArchiveJobJson.Default.ImageJobParameters), JobPriority.Background);
|
||||
|
||||
public static ArchiveJobRequest Maintenance(Archive archive, bool dryRun = false) =>
|
||||
new(JobKind.Maintenance, [archive.Id], archive.Name, Json(new MaintenanceParameters(dryRun), ArchiveJobJson.Default.MaintenanceParameters),
|
||||
JobPriority.Normal);
|
||||
|
||||
public static Task<Job> EnqueueAsync(this JobEngine engine, ArchiveJobRequest request, Guid? createdBy = null, CancellationToken cancellationToken = default) =>
|
||||
engine.EnqueueAsync(request.Kind, request.Targets, request.Label, request.ParametersJson, request.Priority, createdBy, cancellationToken);
|
||||
|
||||
private static string Json<T>(T value, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo) => JsonSerializer.Serialize(value, typeInfo);
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
using System.Diagnostics;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>A frame whose original is missing, changed or damaged.</summary>
|
||||
public sealed record VerifyProblem(Guid SequenceId, Guid FrameId, string RelativePath, FrameHealth Health);
|
||||
|
||||
/// <summary>Where a verify stands: the next sequence (ordinal in the run) and the next frame index in it.</summary>
|
||||
public readonly record struct VerifyPosition(int Sequence, int Frame);
|
||||
|
||||
/// <summary>Outcome of one verify run (frames before a resume point are counted by the run that checked them).</summary>
|
||||
public sealed record VerifyReport(int Sequences, long Checked, long Ok, long Hashed, IReadOnlyList<VerifyProblem> Problems);
|
||||
|
||||
/// <summary>
|
||||
/// Re-hashes originals against their sidecar SHA-256 (BRIEF §12.5 Verify): missing, changed and corrupt files get
|
||||
/// <see cref="FrameFlags.Missing"/>/<see cref="FrameFlags.Corrupt"/> in the sidecar and the index, adopted files get
|
||||
/// their first hash, and a clean sequence is stamped verified. Progress is flushed in batches so a resumed run
|
||||
/// continues where the last flush ended.
|
||||
/// </summary>
|
||||
public sealed class ArchiveVerifier
|
||||
{
|
||||
private const int FlushFrames = 512;
|
||||
private static readonly TimeSpan FlushInterval = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
|
||||
internal ArchiveVerifier(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
/// <param name="sequenceIds">Sequences to verify; null = the whole archive (ordered by folder).</param>
|
||||
/// <param name="checkpoint">Called after each flush with the resume position, bytes done and bytes total.</param>
|
||||
/// <param name="frameDone">Called after each frame with bytes done and bytes total (throughput and ETA).</param>
|
||||
public async Task<VerifyReport> VerifyAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, VerifyPosition resume,
|
||||
Func<VerifyPosition, long, long, ValueTask> checkpoint, Func<long, long, ValueTask>? frameDone = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
var targets = await TargetsAsync(archive, sequenceIds, cancellationToken).ConfigureAwait(false);
|
||||
var bytesTotal = targets.Sum(t => t.TotalBytes);
|
||||
var bytesDone = targets.Take(resume.Sequence).Sum(t => t.TotalBytes);
|
||||
long checkedFrames = 0, ok = 0, hashed = 0;
|
||||
var problems = new List<VerifyProblem>();
|
||||
for (var s = resume.Sequence; s < targets.Count; s++)
|
||||
{
|
||||
var target = targets[s];
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
var pending = new List<FrameCheck>();
|
||||
var clock = Stopwatch.StartNew();
|
||||
var first = s == resume.Sequence ? System.Math.Min(resume.Frame, manifest.Frames.Count) : 0;
|
||||
bytesDone += manifest.Frames.Take(first).Sum(f => f.Size);
|
||||
for (var i = first; i < manifest.Frames.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var check = await FrameVerifier.CheckAsync(sequence, manifest.Frames[i], read => bytesDone += read, cancellationToken).ConfigureAwait(false);
|
||||
checkedFrames++;
|
||||
if (frameDone is not null) await frameDone(bytesDone, bytesTotal).ConfigureAwait(false);
|
||||
if (check.Health == FrameHealth.Ok) ok++;
|
||||
else if (check.Health == FrameHealth.Hashed) hashed++;
|
||||
else problems.Add(new VerifyProblem(target.Id, check.Frame.Id, sequence.ArchiveRelative(check.Frame.File), check.Health));
|
||||
if (check.ChangesRecord) pending.Add(check);
|
||||
if (i + 1 < manifest.Frames.Count && (pending.Count >= FlushFrames || clock.Elapsed >= FlushInterval))
|
||||
{
|
||||
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
|
||||
await checkpoint(new VerifyPosition(s, i + 1), bytesDone, bytesTotal).ConfigureAwait(false);
|
||||
clock.Restart();
|
||||
}
|
||||
}
|
||||
await FlushAsync(archive, paths, target, pending, cancellationToken).ConfigureAwait(false);
|
||||
await StampAsync(archive, paths, target, cancellationToken).ConfigureAwait(false);
|
||||
await checkpoint(new VerifyPosition(s + 1, 0), bytesDone, bytesTotal).ConfigureAwait(false);
|
||||
}
|
||||
return new VerifyReport(targets.Count, checkedFrames, ok, hashed, problems);
|
||||
}
|
||||
|
||||
private async Task<List<Sequence>> TargetsAsync(Archive archive, IReadOnlyList<Guid>? sequenceIds, CancellationToken cancellationToken)
|
||||
{
|
||||
var ids = sequenceIds ?? (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Id).ToList();
|
||||
var targets = new List<Sequence>(ids.Count);
|
||||
foreach (var id in ids)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is { } s && s.ArchiveId == archive.Id) targets.Add(s);
|
||||
}
|
||||
return targets;
|
||||
}
|
||||
|
||||
/// <summary>Writes the flags and new hashes of checked frames to the sidecar and the index.</summary>
|
||||
private async Task FlushAsync(Archive archive, ArchivePaths paths, Sequence target, List<FrameCheck> pending, CancellationToken cancellationToken)
|
||||
{
|
||||
if (pending.Count == 0) return;
|
||||
var changes = pending.ToDictionary(c => c.Frame.Id, c => (Func<SidecarFrame, SidecarFrame>)(f =>
|
||||
f with { Flags = c.Flags, Sha256 = c.Health == FrameHealth.Hashed ? c.Sha256 : f.Sha256 }));
|
||||
await _writer.UpdateAsync(paths, target.Id, target.Folder, m => SequenceFrames.Update(m, changes), cancellationToken).ConfigureAwait(false);
|
||||
await _db.Frames.UpdateIntegrityAsync(pending.ConvertAll(c => new FrameIntegrity(c.Frame.Id, c.Flags, c.Health == FrameHealth.Hashed ? c.Sha256 : null)),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, paths.Sequence(target.Folder), target.Id, cancellationToken).ConfigureAwait(false);
|
||||
pending.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Stamps the verify time; the sequence counts as verified only when no frame is missing or corrupt.</summary>
|
||||
private async Task StampAsync(Archive archive, ArchivePaths paths, Sequence target, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = paths.Sequence(target.Folder);
|
||||
var manifest = await _writer.UpdateAsync(paths, target.Id, target.Folder, m =>
|
||||
{
|
||||
var clean = m.Frames.All(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) == 0);
|
||||
return m with
|
||||
{
|
||||
Status = clean ? m.Status | SequenceStatus.Verified : m.Status & ~SequenceStatus.Verified,
|
||||
VerifiedAt = DateTimeOffset.UtcNow,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
var existing = await _db.Sequences.GetAsync(target.Id, cancellationToken).ConfigureAwait(false);
|
||||
await _db.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
await _indexer.RecordStampsAsync(archive, sequence, target.Id, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Metadata;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Core.Naming;
|
||||
using Titano.Data.Archiving.Jobs;
|
||||
using Titano.Data.Repositories;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>A card volume as the client sees it (BRIEF §13): identity <c>{serial}-{label}-{capacity}</c>.</summary>
|
||||
public sealed record CardVolume(string Identity, uint SerialNumber, string? Label, long Capacity);
|
||||
|
||||
public enum AfterImport
|
||||
{
|
||||
Keep,
|
||||
|
||||
/// <summary>Delete from the card the photos whose card file and archive copy both re-hash to the imported hash.</summary>
|
||||
DeleteImported,
|
||||
|
||||
/// <summary>Offer the operating system's format dialog (only after a fully verified import).</summary>
|
||||
FormatCard,
|
||||
}
|
||||
|
||||
/// <summary>Import defaults remembered per card (<see cref="MediaCard.ProfileJson"/>, BRIEF §12.4).</summary>
|
||||
public sealed record CardProfile
|
||||
{
|
||||
public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
|
||||
public string FileTemplate { get; init; } = TemplateDefaults.File;
|
||||
public int SessionGapMinutes { get; init; } = 45;
|
||||
public int MinimumSessionFrames { get; init; } = 8;
|
||||
public bool IncludeSubfolders { get; init; } = true;
|
||||
public bool VerifyCopies { get; init; } = true;
|
||||
public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
|
||||
public AfterImport AfterImport { get; init; } = AfterImport.Keep;
|
||||
public bool Eject { get; init; } = true;
|
||||
public bool DeepAnalysis { get; init; } = true;
|
||||
|
||||
public ImportOptions ToOptions() => new()
|
||||
{
|
||||
FolderTemplate = FolderTemplate,
|
||||
FileTemplate = FileTemplate,
|
||||
SessionGap = TimeSpan.FromMinutes(SessionGapMinutes),
|
||||
MinimumSessionFrames = MinimumSessionFrames,
|
||||
IncludeSubfolders = IncludeSubfolders,
|
||||
VerifyCopies = VerifyCopies,
|
||||
Duplicates = Duplicates,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Card import over the archive index (BRIEF §12.4, §13): card profiles, planning with the archive's history and
|
||||
/// duplicates, execution into the archive through <see cref="ArchiveIngest"/>, and the verified card erase.
|
||||
/// </summary>
|
||||
public sealed class ImportService
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly ArchiveIngest _ingest;
|
||||
|
||||
internal ImportService(TitanoDatabase db, ArchiveIngest ingest)
|
||||
{
|
||||
_db = db;
|
||||
_ingest = ingest;
|
||||
}
|
||||
|
||||
/// <summary>The card with this identity, registered on first sight (the UI then asks for a nickname).</summary>
|
||||
public async Task<MediaCard> RegisterCardAsync(CardVolume volume, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
if (await _db.MediaCards.FindByIdentityAsync(volume.Identity, cancellationToken).ConfigureAwait(false) is { } known)
|
||||
{
|
||||
await _db.MediaCards.TouchAsync(known.Id, now, cancellationToken).ConfigureAwait(false);
|
||||
return known with { LastSeen = now };
|
||||
}
|
||||
var card = new MediaCard
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
Identity = volume.Identity,
|
||||
VolumeSerial = volume.SerialNumber,
|
||||
Label = volume.Label,
|
||||
Capacity = volume.Capacity,
|
||||
FirstSeen = now,
|
||||
LastSeen = now,
|
||||
};
|
||||
await _db.MediaCards.UpsertAsync(card, cancellationToken).ConfigureAwait(false);
|
||||
return card;
|
||||
}
|
||||
|
||||
public static CardProfile ProfileOf(MediaCard card) =>
|
||||
card.ProfileJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.CardProfile) ?? new() : new();
|
||||
|
||||
/// <summary>Stores the nickname, defaults and auto-import choice of a card.</summary>
|
||||
public async Task<MediaCard> SaveCardAsync(Guid cardId, string? nickname, CardProfile profile, bool autoImport, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var card = await _db.MediaCards.GetAsync(cardId, cancellationToken).ConfigureAwait(false) ?? throw new ArgumentException($"Unknown card {cardId}.", nameof(cardId));
|
||||
var updated = card with
|
||||
{
|
||||
Nickname = string.IsNullOrWhiteSpace(nickname) ? null : nickname.Trim(),
|
||||
ProfileJson = JsonSerializer.Serialize(profile, ArchiveJobJson.Default.CardProfile),
|
||||
AutoImport = autoImport,
|
||||
};
|
||||
await _db.MediaCards.UpsertAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
return updated;
|
||||
}
|
||||
|
||||
/// <summary>Scans the source, reads light metadata and plans the import into the archive. Nothing is written
|
||||
/// except the card's camera identity on its first plan.</summary>
|
||||
/// <param name="progress">Photos read so far.</param>
|
||||
public async Task<ImportPlan> PlanAsync(Archive archive, string sourceRoot, ImportOptions options, Guid? cardId, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var problems = new List<ImportProblem>();
|
||||
var files = CardScanner.Find(sourceRoot, options.IncludeSubfolders, cancellationToken);
|
||||
var photos = CardScanner.ReadMetadata(sourceRoot, files, problems, progress, cancellationToken);
|
||||
var card = cardId is { } id ? await _db.MediaCards.GetAsync(id, cancellationToken).ConfigureAwait(false) : null;
|
||||
if (card is { CameraModel: null } && photos.Count > 0)
|
||||
{
|
||||
var first = photos[0].Metadata;
|
||||
await _db.MediaCards.UpsertAsync(card with { CameraMake = first.Make, CameraModel = first.Model, CameraSerial = first.BodySerial },
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
var imported = card is null ? new HashSet<ImportKey>() : await _db.MediaCards.GetImportedKeysAsync(card.Id, cancellationToken).ConfigureAwait(false);
|
||||
var folders = (await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false)).Select(l => l.Folder)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var lookup = new IndexLookup(_db, archive.Id, paths, imported, folders);
|
||||
return await ImportPlanner.PlanAsync(sourceRoot, paths.Root, photos, problems, options, lookup, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Runs (or resumes) a confirmed plan; <paramref name="committed"/> sees each batch after its history is recorded.</summary>
|
||||
internal async Task<ImportProgress> ExecuteAsync(ImportJobParameters job, Guid jobId, int startAt, Func<ImportBatch, ValueTask> committed,
|
||||
Func<ImportProgress, ValueTask>? progress, CancellationToken cancellationToken)
|
||||
{
|
||||
var archive = await _db.Archives.GetAsync(job.ArchiveId, cancellationToken).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException($"Unknown archive {job.ArchiveId}.");
|
||||
ArchiveVolume.EnsureWritable(new ArchivePaths(archive.RootPath));
|
||||
var sink = new IngestSink(_ingest, archive, job.UserId?.ToString());
|
||||
return await new ImportExecutor(job.Plan, sink).RunAsync(startAt, job.Verify, async batch =>
|
||||
{
|
||||
if (job.CardId is { } card) await RecordHistoryAsync(card, jobId, batch, cancellationToken).ConfigureAwait(false);
|
||||
await committed(batch).ConfigureAwait(false);
|
||||
}, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Dry run of "delete imported": every card file with a recorded import, checked against its archive copy.</summary>
|
||||
public async Task<ErasePlan> PlanEraseAsync(Archive archive, Guid cardId, string sourceRoot, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var candidates = new List<EraseCandidate>();
|
||||
foreach (var record in await _db.MediaCards.ListVerifiableImportsAsync(cardId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (!File.Exists(Path.Combine(sourceRoot, record.Key.Path))) continue;
|
||||
if (await _db.Frames.GetAsync(record.FrameId!.Value, cancellationToken).ConfigureAwait(false) is not { } frame) continue;
|
||||
candidates.Add(new EraseCandidate(record.Key.Path, record.Key.Size, record.Sha256!, paths.FromRelative(frame.RelativePath)));
|
||||
}
|
||||
return await CardEraser.PlanAsync(sourceRoot, candidates, progress, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private Task RecordHistoryAsync(Guid cardId, Guid jobId, ImportBatch batch, CancellationToken cancellationToken)
|
||||
{
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var records = new List<ImportRecord>();
|
||||
foreach (var r in batch.Results)
|
||||
{
|
||||
var key = new ImportKey(r.Item.SourcePath, r.Item.Size, new DateTimeOffset(DateTime.SpecifyKind(r.Item.ModifiedUtc, DateTimeKind.Utc)));
|
||||
switch (r.Status)
|
||||
{
|
||||
case ImportItemStatus.Copied or ImportItemStatus.Duplicate:
|
||||
records.Add(new ImportRecord(cardId, key, r.Sha256, r.Commit?.FrameId, jobId, now));
|
||||
break;
|
||||
case ImportItemStatus.Skipped when r.Item.Action == ImportAction.SkipInArchive:
|
||||
records.Add(new ImportRecord(cardId, key, r.Item.Sha256, r.Item.ExistingFrameId, jobId, now));
|
||||
break;
|
||||
}
|
||||
}
|
||||
return records.Count == 0 ? Task.CompletedTask : _db.MediaCards.RecordImportsAsync(records, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Planner questions answered from the index and the archive folder.</summary>
|
||||
private sealed class IndexLookup(TitanoDatabase db, Guid archiveId, ArchivePaths paths, IReadOnlySet<ImportKey> imported, HashSet<string> indexedFolders)
|
||||
: IImportLookup
|
||||
{
|
||||
public Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(imported.Contains(new ImportKey(file.RelativePath, file.Size, new DateTimeOffset(DateTime.SpecifyKind(file.ModifiedUtc, DateTimeKind.Utc)))));
|
||||
|
||||
public async Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken) =>
|
||||
(await db.Frames.FindBySizeAndCaptureAsync(archiveId, size, capturedAt, cancellationToken).ConfigureAwait(false))
|
||||
.Select(f => new ArchiveMatch(f.Id, f.Sha256)).ToList();
|
||||
|
||||
public async Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) =>
|
||||
await db.Sequences.FindBySessionKeyAsync(archiveId, sessionKey, cancellationToken).ConfigureAwait(false) is { } id
|
||||
? (await db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false))?.Folder
|
||||
: null;
|
||||
|
||||
public bool IsFolderTaken(string folder)
|
||||
{
|
||||
if (indexedFolders.Contains(folder)) return true;
|
||||
var path = paths.Sequence(folder).Folder;
|
||||
return Directory.Exists(path) && Directory.EnumerateFileSystemEntries(path).Any();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Import destination "Archive": each plan session becomes (or continues) the sequence of its session key.</summary>
|
||||
private sealed class IngestSink(ArchiveIngest ingest, Archive archive, string? createdBy) : IImportSink
|
||||
{
|
||||
public string StagingDirectory { get; } = new ArchivePaths(archive.RootPath).StagingDirectory;
|
||||
|
||||
public async Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken)
|
||||
{
|
||||
var frames = files.Select(f => new StagedFrame(f.StagedPath, Path.GetFileName(f.Item.SourcePath), f.Item.Size, f.Sha256,
|
||||
MetadataReader.Read(f.StagedPath, cancellationToken), f.Item.CapturedAt, f.Item.FileName)).ToList();
|
||||
var results = await ingest.CommitSessionAsync(archive, new SequenceTarget(session.SessionKey, session.Title, session.Folder, createdBy),
|
||||
frames, cancellationToken).ConfigureAwait(false);
|
||||
return results.Select(r => new SinkCommit(r.FrameId, r.RelativePath, r.Duplicate)).ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary><see cref="JobKind.Scan"/>: target = the archive.</summary>
|
||||
public sealed record ScanParameters(bool Full);
|
||||
|
||||
/// <summary><see cref="JobKind.Verify"/>: target = the archive; null sequences = all of them.</summary>
|
||||
public sealed record VerifyParameters(IReadOnlyList<Guid>? SequenceIds);
|
||||
|
||||
/// <summary><see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/>: targets = sequences.</summary>
|
||||
public sealed record ImageJobParameters(bool Force);
|
||||
|
||||
/// <summary><see cref="JobKind.Maintenance"/>: target = the archive.</summary>
|
||||
public sealed record MaintenanceParameters(bool DryRun);
|
||||
|
||||
/// <summary><see cref="JobKind.Import"/>: the exact plan the user confirmed.</summary>
|
||||
public sealed record ImportJobParameters(Guid ArchiveId, ImportPlan Plan, Guid? CardId, bool Verify, Guid? UserId);
|
||||
|
||||
/// <summary>A job ready to enqueue.</summary>
|
||||
public sealed record ArchiveJobRequest(JobKind Kind, IReadOnlyList<Guid> Targets, string? Label, string? ParametersJson, JobPriority Priority);
|
||||
|
||||
[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, UseStringEnumConverter = true)]
|
||||
[JsonSerializable(typeof(ScanParameters))]
|
||||
[JsonSerializable(typeof(VerifyParameters))]
|
||||
[JsonSerializable(typeof(ImageJobParameters))]
|
||||
[JsonSerializable(typeof(MaintenanceParameters))]
|
||||
[JsonSerializable(typeof(ImportJobParameters))]
|
||||
[JsonSerializable(typeof(CardProfile))]
|
||||
internal sealed partial class ArchiveJobJson : JsonSerializerContext
|
||||
{
|
||||
/// <summary>Parameters of a job, or <paramref name="fallback"/> when it has none.</summary>
|
||||
public static T Read<T>(Job job, System.Text.Json.Serialization.Metadata.JsonTypeInfo<T> typeInfo, T fallback) =>
|
||||
job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, typeInfo) ?? fallback : fallback;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Thumbs"/> and <see cref="JobKind.Proxies"/> over the target sequences. Progress counts
|
||||
/// sequences; checkpoint: <c>sequence:frame</c>.
|
||||
/// </summary>
|
||||
internal sealed class ImageJobHandler(TitanoDatabase db, ArchiveImages images, JobKind kind) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => kind;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var force = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ImageJobParameters, new ImageJobParameters(false)).Force;
|
||||
var (first, firstFrame) = Parse(context.Checkpoint);
|
||||
var targets = context.Job.TargetIds;
|
||||
for (var s = first; s < targets.Count; s++)
|
||||
{
|
||||
context.Cancellation.ThrowIfCancellationRequested();
|
||||
if (await db.Sequences.GetAsync(targets[s], context.Cancellation).ConfigureAwait(false) is not { } sequence
|
||||
|| await db.Archives.GetAsync(sequence.ArchiveId, context.Cancellation).ConfigureAwait(false) is not { } archive)
|
||||
{
|
||||
context.Log(Loc.L("Archive.Log.SequenceMissing", targets[s]));
|
||||
continue;
|
||||
}
|
||||
var index = s;
|
||||
ValueTask Checkpoint(int nextFrame, int frames) =>
|
||||
context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{index}:{nextFrame}"), index, targets.Count);
|
||||
var start = s == first ? firstFrame : 0;
|
||||
var result = kind == JobKind.Thumbs
|
||||
? await images.BuildThumbnailsAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false)
|
||||
: await images.BuildProxiesAsync(archive, sequence, force, start, Checkpoint, context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ImagesDone", sequence.DisplayTitle, result.Built, result.Reused, result.Undecodable));
|
||||
await context.CheckpointAsync(string.Create(CultureInfo.InvariantCulture, $"{s + 1}:0"), s + 1, targets.Count).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static (int Sequence, int Frame) Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':');
|
||||
return parts is [var s, var f]
|
||||
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
|
||||
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
|
||||
? (sequence, frame)
|
||||
: (0, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Import"/>: executes the confirmed plan in its parameters; progress in bytes. Checkpoint: the
|
||||
/// number of plan items done. A removed card fails the job at a batch boundary; resuming it after re-insertion
|
||||
/// continues there.
|
||||
/// </summary>
|
||||
internal sealed class ImportJobHandler(ImportService imports) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => JobKind.Import;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var parameters = (context.Job.ParametersJson is { Length: > 0 } json ? JsonSerializer.Deserialize(json, ArchiveJobJson.Default.ImportJobParameters) : null)
|
||||
?? throw new InvalidOperationException($"Import job {context.Job.Id} has no plan.");
|
||||
var start = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var next) ? next : 0;
|
||||
var final = await imports.ExecuteAsync(parameters, context.Job.Id, start, async batch =>
|
||||
{
|
||||
foreach (var failed in batch.Results.Where(r => r.Status == ImportItemStatus.Failed))
|
||||
context.Log(Loc.L("Archive.Log.ImportFailed", failed.Item.SourcePath, failed.Error));
|
||||
await context.CheckpointAsync(batch.NextItem.ToString(CultureInfo.InvariantCulture), batch.Progress.BytesDone, batch.Progress.BytesTotal)
|
||||
.ConfigureAwait(false);
|
||||
}, p => context.ReportAsync(p.BytesDone, p.BytesTotal, p.Current), context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.ImportDone", final.Copied, final.Skipped, final.Failed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Maintenance"/>: stale leases, leftover temporary files, thumbnail pack compaction, duplicate
|
||||
/// flags, then an orphan count. Checkpoint: the number of steps done.
|
||||
/// </summary>
|
||||
internal sealed class MaintenanceJobHandler(TitanoDatabase db, ArchiveMaintenance maintenance) : IJobHandler
|
||||
{
|
||||
public JobKind Kind => JobKind.Maintenance;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
|
||||
var dryRun = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.MaintenanceParameters, new MaintenanceParameters(false)).DryRun;
|
||||
var first = int.TryParse(context.Checkpoint, NumberStyles.None, CultureInfo.InvariantCulture, out var step) ? step : 0;
|
||||
var report = await maintenance.RunAsync(archive, dryRun, first,
|
||||
done => context.CheckpointAsync(done.ToString(CultureInfo.InvariantCulture), done, ArchiveMaintenance.StepCount),
|
||||
context.Cancellation).ConfigureAwait(false);
|
||||
if (dryRun) context.Log(Loc.L("Archive.Log.DryRun"));
|
||||
context.Log(Loc.L("Archive.Log.MaintenanceDone", report.StaleLeases.Count, report.TemporaryFiles.Count, report.CompactedPacks.Count,
|
||||
report.DuplicateFlagChanges));
|
||||
context.Log(Loc.L("Archive.Log.Orphans", report.UnindexedFiles, report.MissingFiles));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary>Resolves the archive a job targets (its first target id).</summary>
|
||||
internal static class JobArchive
|
||||
{
|
||||
public static async Task<Archive> LoadAsync(TitanoDatabase db, Job job, CancellationToken cancellationToken) =>
|
||||
job.TargetIds.Count > 0 && await db.Archives.GetAsync(job.TargetIds[0], cancellationToken).ConfigureAwait(false) is { } archive
|
||||
? archive
|
||||
: throw new InvalidOperationException($"{job.Kind} job {job.Id} does not target a known archive.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="JobKind.Scan"/> (incremental, or full with <see cref="ScanParameters.Full"/>) and
|
||||
/// <see cref="JobKind.Rebuild"/>. Checkpoint: <c>after:<folder></c>, the last folder indexed.
|
||||
/// </summary>
|
||||
internal sealed class ScanJobHandler(TitanoDatabase db, ArchiveIndexer indexer, JobKind kind) : IJobHandler
|
||||
{
|
||||
private const string Prefix = "after:";
|
||||
|
||||
public JobKind Kind => kind;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
|
||||
var mode = kind == JobKind.Rebuild ? ScanMode.Rebuild
|
||||
: ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.ScanParameters, new ScanParameters(false)).Full ? ScanMode.Full
|
||||
: ScanMode.Incremental;
|
||||
var resume = context.Checkpoint is { } checkpoint && checkpoint.StartsWith(Prefix, StringComparison.Ordinal) ? checkpoint[Prefix.Length..] : null;
|
||||
var summary = await indexer.ScanAsync(archive, mode, resume, async p =>
|
||||
{
|
||||
if (p.Changed) await context.CheckpointAsync(Prefix + p.Folder, p.FoldersDone, p.FoldersTotal).ConfigureAwait(false);
|
||||
else await context.ReportAsync(p.FoldersDone, p.FoldersTotal, p.Folder).ConfigureAwait(false);
|
||||
}, context.Cancellation).ConfigureAwait(false);
|
||||
foreach (var (folder, error) in summary.Failed) context.Log(Loc.L("Archive.Log.FolderFailed", folder, error));
|
||||
context.Log(Loc.L("Archive.Log.ScanDone", summary.Folders, summary.Indexed, summary.Adopted, summary.Removed));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System.Globalization;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
using Titano.Data.Jobs;
|
||||
|
||||
namespace Titano.Data.Archiving.Jobs;
|
||||
|
||||
/// <summary><see cref="JobKind.Verify"/>; progress in bytes hashed. Checkpoint: <c>sequence:frame</c>.</summary>
|
||||
internal sealed class VerifyJobHandler(TitanoDatabase db, ArchiveVerifier verifier) : IJobHandler
|
||||
{
|
||||
/// <summary>Problems listed one per log line; the rest are counted (the index flags them all).</summary>
|
||||
private const int LoggedProblems = 25;
|
||||
|
||||
public JobKind Kind => JobKind.Verify;
|
||||
|
||||
public async Task RunAsync(JobContext context)
|
||||
{
|
||||
var archive = await JobArchive.LoadAsync(db, context.Job, context.Cancellation).ConfigureAwait(false);
|
||||
var parameters = ArchiveJobJson.Read(context.Job, ArchiveJobJson.Default.VerifyParameters, new VerifyParameters(null));
|
||||
var report = await verifier.VerifyAsync(archive, parameters.SequenceIds, Parse(context.Checkpoint),
|
||||
(position, done, total) => context.CheckpointAsync(
|
||||
string.Create(CultureInfo.InvariantCulture, $"{position.Sequence}:{position.Frame}"), done, total),
|
||||
(done, total) => context.ReportAsync(done, total),
|
||||
context.Cancellation).ConfigureAwait(false);
|
||||
foreach (var problem in report.Problems.Take(LoggedProblems))
|
||||
{
|
||||
context.Log(problem.Health switch
|
||||
{
|
||||
FrameHealth.Missing => Loc.L("Archive.Log.FrameMissing", problem.RelativePath),
|
||||
FrameHealth.Changed => Loc.L("Archive.Log.FrameChanged", problem.RelativePath),
|
||||
_ => Loc.L("Archive.Log.FrameCorrupt", problem.RelativePath),
|
||||
});
|
||||
}
|
||||
context.Log(Loc.L("Archive.Log.VerifyDone", report.Checked, report.Ok, report.Hashed, report.Problems.Count));
|
||||
}
|
||||
|
||||
private static VerifyPosition Parse(string? checkpoint)
|
||||
{
|
||||
var parts = checkpoint?.Split(':');
|
||||
return parts is [var s, var f]
|
||||
&& int.TryParse(s, NumberStyles.None, CultureInfo.InvariantCulture, out var sequence)
|
||||
&& int.TryParse(f, NumberStyles.None, CultureInfo.InvariantCulture, out var frame)
|
||||
? new VerifyPosition(sequence, frame)
|
||||
: default;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Localization;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>The plan no longer matches the archive (a sequence moved or vanished since it was made).</summary>
|
||||
public sealed class StaleRenamePlanException(Guid sequenceId)
|
||||
: InvalidOperationException(Loc.L("Archive.Error.StalePlan"))
|
||||
{
|
||||
public Guid SequenceId { get; } = sequenceId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mass rename of sequence folders and titles (BRIEF §7, §12.5): <see cref="PlanAsync"/> is the dry run;
|
||||
/// <see cref="ApplyAsync"/> makes exactly the planned changes to folders, sidecars and the index, step by step, with a
|
||||
/// journal on the archive so an interrupted rename is completed by <see cref="ResumePendingAsync"/>.
|
||||
/// </summary>
|
||||
public sealed class MassRenamer
|
||||
{
|
||||
private readonly TitanoDatabase _db;
|
||||
private readonly SequenceWriter _writer;
|
||||
private readonly ArchiveIndexer _indexer;
|
||||
|
||||
internal MassRenamer(TitanoDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
||||
{
|
||||
_db = db;
|
||||
_writer = writer;
|
||||
_indexer = indexer;
|
||||
}
|
||||
|
||||
/// <param name="folderTemplate">New folder template (<c>{title}</c> = the new title), or null to keep folders.</param>
|
||||
/// <param name="titleTemplate">New title template (<c>{title}</c> = the current title), or null to keep titles.</param>
|
||||
public async Task<RenamePlan> PlanAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, string? folderTemplate, string? titleTemplate,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var locations = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
||||
var subjects = new List<RenameSubject>(sequenceIds.Count);
|
||||
foreach (var id in sequenceIds)
|
||||
{
|
||||
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } s || s.ArchiveId != archive.Id)
|
||||
throw new ArgumentException($"Sequence {id} is not in archive {archive.Id}.", nameof(sequenceIds));
|
||||
subjects.Add(new RenameSubject(s.Id, s.Folder, s.Title, s.FirstCapture, s.CameraMake, s.CameraModel, s.Lens));
|
||||
}
|
||||
return RenamePlanner.Plan(archive.Id, subjects, folderTemplate, titleTemplate, locations.Select(l => l.Folder),
|
||||
folder => Directory.Exists(paths.Sequence(folder).Folder));
|
||||
}
|
||||
|
||||
/// <summary>Applies a plan; each completed step is journaled before the next starts.</summary>
|
||||
/// <exception cref="StaleRenamePlanException">A sequence is no longer where the plan expects it; nothing of that step was changed.</exception>
|
||||
public Task ApplyAsync(Archive archive, RenamePlan plan, IProgress<int>? progress = null, CancellationToken cancellationToken = default) =>
|
||||
RunAsync(archive, new RenameJournal(plan, []), resuming: false, progress, cancellationToken);
|
||||
|
||||
/// <summary>Completes renames interrupted by a crash; returns how many plans were resumed.</summary>
|
||||
public async Task<int> ResumePendingAsync(Archive archive, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
var pending = RenameJournal.ListPending(paths).Where(j => j.Plan.ArchiveId == archive.Id).ToList();
|
||||
foreach (var journal in pending) await RunAsync(archive, journal, resuming: true, null, cancellationToken).ConfigureAwait(false);
|
||||
return pending.Count;
|
||||
}
|
||||
|
||||
/// <param name="resuming">Steps whose sequence the index no longer knows are dropped instead of failing the rename:
|
||||
/// their sidecars still carry the ids, so the next scan indexes them wherever they ended up.</param>
|
||||
private async Task RunAsync(Archive archive, RenameJournal journal, bool resuming, IProgress<int>? progress, CancellationToken cancellationToken)
|
||||
{
|
||||
var paths = new ArchivePaths(archive.RootPath);
|
||||
ArchiveVolume.EnsureWritable(paths);
|
||||
if (journal.Plan.ArchiveId != archive.Id) throw new ArgumentException("The plan belongs to another archive.", nameof(journal));
|
||||
var completed = journal.Completed.ToHashSet();
|
||||
if (!resuming)
|
||||
{
|
||||
// A plan made before another change is refused before anything moves.
|
||||
foreach (var step in journal.Plan.Steps)
|
||||
{
|
||||
if ((await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false))?.Folder != step.FromFolder)
|
||||
throw new StaleRenamePlanException(step.SequenceId);
|
||||
}
|
||||
}
|
||||
RenameJournal.Write(paths, journal);
|
||||
for (var i = 0; i < journal.Plan.Steps.Count; i++)
|
||||
{
|
||||
var step = journal.Plan.Steps[i];
|
||||
if (completed.Contains(step.SequenceId)) continue;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
await ApplyStepAsync(archive, paths, step, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (StaleRenamePlanException) when (resuming)
|
||||
{
|
||||
Log.Warning("archive", $"Rename of {step.FromFolder} dropped: the index no longer has that sequence there");
|
||||
}
|
||||
completed.Add(step.SequenceId);
|
||||
RenameJournal.Write(paths, journal with { Completed = [.. completed] });
|
||||
progress?.Report(i + 1);
|
||||
}
|
||||
RenameJournal.Delete(paths, journal.Plan.Id);
|
||||
}
|
||||
|
||||
/// <summary>Folder, then sidecar title, then index; each part is idempotent so a resumed step can repeat it.</summary>
|
||||
private async Task ApplyStepAsync(Archive archive, ArchivePaths paths, RenameStep step, CancellationToken cancellationToken)
|
||||
{
|
||||
await using (await _writer.LockAsync(paths, step.SequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var row = await _db.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
if (row is null || (row.Folder != step.FromFolder && row.Folder != step.ToFolder)) throw new StaleRenamePlanException(step.SequenceId);
|
||||
RenamePlanner.MoveFolder(paths, step);
|
||||
var sequence = paths.Sequence(step.ToFolder);
|
||||
var manifest = SequenceWriter.ReadLocked(sequence, step.SequenceId);
|
||||
if (manifest.Title != step.ToTitle)
|
||||
{
|
||||
manifest = manifest with
|
||||
{
|
||||
Title = step.ToTitle,
|
||||
Versions = manifest.Versions with { Settings = manifest.Versions.Settings + 1 },
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
};
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, manifest);
|
||||
}
|
||||
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var existing = await tx.Sequences.GetAsync(step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Sequences.UpsertAsync(SequenceMapping.ToSequence(manifest, archive.Id, sequence, existing), cancellationToken).ConfigureAwait(false);
|
||||
if (step.MovesFolder) await tx.Frames.RebaseFolderAsync(step.SequenceId, step.FromFolder, step.ToFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.Archives.DeleteFolderStateAsync(archive.Id, step.FromFolder, cancellationToken).ConfigureAwait(false);
|
||||
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
await _indexer.RecordStampsAsync(archive, sequence, step.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
using System.Text.Json;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Sidecar records to index rows. Fields the sidecar does not carry are kept from the existing row.</summary>
|
||||
internal static class SequenceMapping
|
||||
{
|
||||
public static Frame ToFrame(SidecarFrame f, Guid sequenceId, SequencePaths sequence) => new()
|
||||
{
|
||||
Id = f.Id,
|
||||
SequenceId = sequenceId,
|
||||
Index = f.I,
|
||||
RelativePath = sequence.ArchiveRelative(f.File),
|
||||
FileName = Path.GetFileName(f.File),
|
||||
OriginalFileName = f.Original,
|
||||
Size = f.Size,
|
||||
Sha256 = f.Sha256,
|
||||
CapturedAt = f.CapturedAt,
|
||||
UtcOffsetMinutes = f.UtcOffset,
|
||||
TimestampSource = f.TimeSource,
|
||||
ExposureTime = f.Exposure,
|
||||
FNumber = f.FNumber,
|
||||
Iso = f.Iso,
|
||||
FocalLength = f.Focal,
|
||||
ExposureBias = f.Bias,
|
||||
WhiteBalance = f.Wb,
|
||||
Orientation = f.Orientation,
|
||||
Width = f.W,
|
||||
Height = f.H,
|
||||
Format = f.Format,
|
||||
PreviewOffset = f.PreviewOffset,
|
||||
PreviewLength = f.PreviewLength,
|
||||
Flags = f.Flags,
|
||||
};
|
||||
|
||||
public static List<Frame> ToFrames(SequenceManifest manifest, SequencePaths sequence) =>
|
||||
manifest.Frames.Select(f => ToFrame(f, manifest.Id, sequence)).ToList();
|
||||
|
||||
/// <param name="existing">The current row; its render, analysis, sync and ownership fields are kept, and so are its
|
||||
/// title, tags, rating and notes when its settings version is ahead of the sidecar's (edited in the index only).</param>
|
||||
public static Sequence ToSequence(SequenceManifest m, Guid archiveId, SequencePaths sequence, Sequence? existing)
|
||||
{
|
||||
var indexAhead = existing is not null && existing.SettingsVersion > m.Versions.Settings;
|
||||
var times = m.Frames.Where(f => f.CapturedAt is not null).Select(f => f.CapturedAt!.Value).Order().ToList();
|
||||
var flagged = m.Frames.Any(f => (f.Flags & (FrameFlags.Missing | FrameFlags.Corrupt)) != 0);
|
||||
return new Sequence
|
||||
{
|
||||
Id = m.Id,
|
||||
ArchiveId = archiveId,
|
||||
Title = indexAhead ? existing!.Title : m.Title,
|
||||
Folder = sequence.RelativeFolder,
|
||||
CameraMake = m.Camera?.Make,
|
||||
CameraModel = m.Camera?.Model,
|
||||
CameraSerial = m.Camera?.Serial,
|
||||
Lens = m.Lens,
|
||||
FirstCapture = times.Count > 0 ? times[0] : null,
|
||||
LastCapture = times.Count > 0 ? times[^1] : null,
|
||||
FrameCount = m.Frames.Count,
|
||||
MedianInterval = MedianInterval(times),
|
||||
TotalBytes = m.Frames.Sum(f => f.Size),
|
||||
Status = flagged ? m.Status | SequenceStatus.HasErrors : m.Status & ~SequenceStatus.HasErrors,
|
||||
VerifiedAt = m.VerifiedAt,
|
||||
Tags = indexAhead ? existing!.Tags : m.Tags,
|
||||
Rating = indexAhead ? existing!.Rating : m.Rating,
|
||||
Notes = indexAhead ? existing!.Notes : m.Notes,
|
||||
SettingsVersion = indexAhead ? existing!.SettingsVersion : m.Versions.Settings,
|
||||
FramesVersion = m.Versions.Frames,
|
||||
UpdatedAt = m.UpdatedAt,
|
||||
RenderedCount = existing?.RenderedCount ?? 0,
|
||||
HeroFrameId = existing?.HeroFrameId,
|
||||
SettingsJson = existing?.SettingsJson ?? ReadProject(sequence),
|
||||
AnalysisJson = existing?.AnalysisJson,
|
||||
Origin = existing?.Origin ?? SequenceOrigin.Local,
|
||||
SyncState = existing?.SyncState ?? SyncState.LocalOnly,
|
||||
OwnerUserId = existing?.OwnerUserId ?? (Guid.TryParse(m.CreatedBy, out var owner) ? owner : null),
|
||||
WarningCount = existing?.WarningCount ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
private static double? MedianInterval(List<DateTime> times)
|
||||
{
|
||||
if (times.Count < 2) return null;
|
||||
var gaps = new double[times.Count - 1];
|
||||
for (var i = 1; i < times.Count; i++) gaps[i - 1] = (times[i] - times[i - 1]).TotalSeconds;
|
||||
Array.Sort(gaps);
|
||||
var mid = gaps.Length / 2;
|
||||
return gaps.Length % 2 == 1 ? gaps[mid] : (gaps[mid - 1] + gaps[mid]) / 2;
|
||||
}
|
||||
|
||||
/// <summary>The project file is owned by the editor; a rebuilt index takes its text as-is when it is valid JSON.</summary>
|
||||
private static string? ReadProject(SequencePaths sequence)
|
||||
{
|
||||
if (!File.Exists(sequence.ProjectPath)) return null;
|
||||
var text = File.ReadAllText(sequence.ProjectPath);
|
||||
try
|
||||
{
|
||||
using (JsonDocument.Parse(text)) return text;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using System.Collections.Concurrent;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Imaging;
|
||||
|
||||
namespace Titano.Data.Archiving;
|
||||
|
||||
/// <summary>Settings shared by the archive services.</summary>
|
||||
public sealed record ArchiveServiceOptions
|
||||
{
|
||||
/// <summary>Decoders for thumbnails and proxies beyond embedded previews and JPEG (WIC on Windows).</summary>
|
||||
public ImageDecoder Decoder { get; init; } = new([]);
|
||||
|
||||
/// <summary>Name written into lease files so a waiting writer can say who holds a sequence.</summary>
|
||||
public string Owner { get; init; } = "titano@" + Environment.MachineName;
|
||||
|
||||
/// <summary>How long a writer waits for a sequence held by another process before giving up.</summary>
|
||||
public TimeSpan LeaseWait { get; init; } = TimeSpan.FromMinutes(2);
|
||||
|
||||
public TimeSpan LeaseDuration { get; init; } = LeaseFile.DefaultDuration;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Read-modify-write of sidecars, serialised per sequence: an in-process gate for this process's writers plus a lease
|
||||
/// file for writers in other processes on the same share (server and client). Every edit sees the latest sidecar,
|
||||
/// with an interrupted ingest batch completed first.
|
||||
/// </summary>
|
||||
internal sealed class SequenceWriter(ArchiveServiceOptions options)
|
||||
{
|
||||
private readonly ConcurrentDictionary<Guid, SemaphoreSlim> _gates = new();
|
||||
|
||||
public ArchiveServiceOptions Options => options;
|
||||
|
||||
/// <summary>Holds the sequence exclusively until disposed.</summary>
|
||||
public async Task<IAsyncDisposable> LockAsync(ArchivePaths archive, Guid sequenceId, CancellationToken cancellationToken)
|
||||
{
|
||||
var gate = _gates.GetOrAdd(sequenceId, _ => new SemaphoreSlim(1, 1));
|
||||
await gate.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var lease = await LeaseFile.AcquireAsync(archive.LocksDirectory, $"seq-{sequenceId:N}", options.Owner, options.LeaseWait,
|
||||
options.LeaseDuration, cancellationToken).ConfigureAwait(false);
|
||||
return new Held(gate, lease);
|
||||
}
|
||||
catch
|
||||
{
|
||||
gate.Release();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the sidecar of a locked sequence, completing an interrupted ingest batch first.
|
||||
/// </summary>
|
||||
/// <exception cref="InvalidDataException">The sidecar in the folder belongs to another sequence.</exception>
|
||||
public static SequenceManifest ReadLocked(SequencePaths sequence, Guid sequenceId)
|
||||
{
|
||||
var manifest = ManifestStore.ReadSequence(sequence.ManifestPath);
|
||||
if (manifest.Id != sequenceId)
|
||||
throw new InvalidDataException($"{sequence.ManifestPath} belongs to sequence {manifest.Id}, not {sequenceId}.");
|
||||
if (IngestJournal.Recover(sequence, manifest) is { } recovered)
|
||||
{
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, recovered);
|
||||
manifest = recovered;
|
||||
}
|
||||
IngestJournal.Delete(sequence);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>Applies <paramref name="edit"/> to the sidecar under the lock; a null result writes nothing.</summary>
|
||||
public async Task<SequenceManifest> UpdateAsync(ArchivePaths archive, Guid sequenceId, string folder,
|
||||
Func<SequenceManifest, SequenceManifest?> edit, CancellationToken cancellationToken)
|
||||
{
|
||||
var sequence = archive.Sequence(folder);
|
||||
await using (await LockAsync(archive, sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var manifest = ReadLocked(sequence, sequenceId);
|
||||
if (edit(manifest) is not { } updated) return manifest;
|
||||
ManifestStore.WriteSequence(sequence.ManifestPath, updated);
|
||||
return updated;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Held(SemaphoreSlim gate, LeaseFile lease) : IAsyncDisposable
|
||||
{
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
lease.Dispose();
|
||||
gate.Release();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Archive index state (BRIEF §7, §14.4). Mirrors sqlite/0002_archive_index.sql.
|
||||
|
||||
-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
|
||||
CREATE TABLE sequence_sessions (
|
||||
archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
session_key text NOT NULL,
|
||||
sequence_id uuid NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (archive_id, session_key)
|
||||
);
|
||||
CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
|
||||
|
||||
-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
|
||||
-- Times are UTC ticks (100 ns), compared exactly with the file system's.
|
||||
CREATE TABLE archive_folders (
|
||||
archive_id uuid NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
folder text COLLATE "C" NOT NULL,
|
||||
sequence_id uuid REFERENCES sequences (id) ON DELETE SET NULL,
|
||||
sidecar_size bigint,
|
||||
sidecar_ticks bigint,
|
||||
photos_ticks bigint,
|
||||
scanned_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (archive_id, folder)
|
||||
);
|
||||
@@ -0,0 +1,23 @@
|
||||
-- Archive index state (BRIEF §7, §14.4). Mirrors postgres/0002_archive_index.sql.
|
||||
|
||||
-- Automatic grouping: a session key (upload session, card session) resolves to the sequence created for it.
|
||||
CREATE TABLE sequence_sessions (
|
||||
archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
session_key TEXT NOT NULL,
|
||||
sequence_id TEXT NOT NULL REFERENCES sequences (id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (archive_id, session_key)
|
||||
);
|
||||
CREATE INDEX ix_sequence_sessions_sequence ON sequence_sessions (sequence_id);
|
||||
|
||||
-- What the last scan saw in each sequence folder, so incremental scans skip unchanged folders by size and time.
|
||||
-- Times are UTC ticks (100 ns), compared exactly with the file system's.
|
||||
CREATE TABLE archive_folders (
|
||||
archive_id TEXT NOT NULL REFERENCES archives (id) ON DELETE CASCADE,
|
||||
folder TEXT NOT NULL,
|
||||
sequence_id TEXT REFERENCES sequences (id) ON DELETE SET NULL,
|
||||
sidecar_size INTEGER,
|
||||
sidecar_ticks INTEGER,
|
||||
photos_ticks INTEGER,
|
||||
scanned_at TEXT NOT NULL,
|
||||
PRIMARY KEY (archive_id, folder)
|
||||
);
|
||||
@@ -1,7 +1,11 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Repositories;
|
||||
|
||||
/// <summary>What the last scan saw in one sequence folder (sidecar and photo-directory stamps).</summary>
|
||||
public sealed record FolderScanState(string Folder, Guid? SequenceId, FileStamp? Sidecar, FileStamp? Photos);
|
||||
|
||||
/// <summary>Archive roots (BRIEF §5, §7).</summary>
|
||||
public sealed class ArchiveRepository
|
||||
{
|
||||
@@ -10,6 +14,9 @@ public sealed class ArchiveRepository
|
||||
private static readonly string UpsertSql = SqlDialect.Upsert("archives",
|
||||
["id", "name", "root_path", "folder_template", "file_template", "schema_version", "created", "proxy_long_edge"], 1);
|
||||
|
||||
private static readonly string FolderStateUpsertSql = SqlDialect.Upsert("archive_folders",
|
||||
["archive_id", "folder", "sequence_id", "sidecar_size", "sidecar_ticks", "photos_ticks", "scanned_at"], 2);
|
||||
|
||||
private readonly DbAccess _db;
|
||||
|
||||
internal ArchiveRepository(DbAccess db) => _db = db;
|
||||
@@ -29,6 +36,31 @@ public sealed class ArchiveRepository
|
||||
public async Task<bool> DeleteAsync(Guid id, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("DELETE FROM archives WHERE id = $1", [id], cancellationToken).ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Scan stamps of every folder of an archive, by folder.</summary>
|
||||
public async Task<IReadOnlyDictionary<string, FolderScanState>> GetFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var rows = await _db.ListAsync(
|
||||
"SELECT folder, sequence_id, sidecar_size, sidecar_ticks, photos_ticks FROM archive_folders WHERE archive_id = $1", [archiveId],
|
||||
r => new FolderScanState(r.GetString(0), r.GetGuidOrNull(1),
|
||||
r.GetInt64OrNull(3) is { } sidecar ? new FileStamp(r.GetInt64(2), new DateTime(sidecar, DateTimeKind.Utc)) : null,
|
||||
r.GetInt64OrNull(4) is { } photos ? new FileStamp(0, new DateTime(photos, DateTimeKind.Utc)) : null),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return rows.ToDictionary(s => s.Folder, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
public async Task SetFolderStateAsync(Guid archiveId, FolderScanState state, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync(FolderStateUpsertSql,
|
||||
[archiveId, state.Folder, state.SequenceId, state.Sidecar?.Size, state.Sidecar?.ModifiedUtc.Ticks, state.Photos?.ModifiedUtc.Ticks, DateTimeOffset.UtcNow],
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
public async Task<bool> DeleteFolderStateAsync(Guid archiveId, string folder, CancellationToken cancellationToken = default) =>
|
||||
await _db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1 AND folder = $2", [archiveId, folder], cancellationToken)
|
||||
.ConfigureAwait(false) > 0;
|
||||
|
||||
/// <summary>Forgets every scan stamp of an archive, so the next scan reads every sidecar (rebuild).</summary>
|
||||
public Task<int> ClearFolderStatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("DELETE FROM archive_folders WHERE archive_id = $1", [archiveId], cancellationToken);
|
||||
|
||||
private static Archive Map(DbRow r) => new()
|
||||
{
|
||||
Id = r.GetGuid(0),
|
||||
|
||||
@@ -4,6 +4,15 @@ using Titano.Core.Model;
|
||||
|
||||
namespace Titano.Data.Repositories;
|
||||
|
||||
/// <summary>Integrity fields a verify rewrites: the flags and the hash of a newly hashed file.</summary>
|
||||
public readonly record struct FrameIntegrity(Guid FrameId, FrameFlags Flags, string? Sha256);
|
||||
|
||||
/// <summary>A frame and the sequence it belongs to.</summary>
|
||||
public readonly record struct FrameRef(Guid FrameId, Guid SequenceId);
|
||||
|
||||
/// <summary>A frame whose content (SHA-256) also exists elsewhere in the archive.</summary>
|
||||
public sealed record DuplicateFrame(string Sha256, Guid FrameId, Guid SequenceId, int Index, string RelativePath);
|
||||
|
||||
/// <summary>Light statistics of one frame, written by the analysis job.</summary>
|
||||
public readonly record struct FrameLightStats(Guid FrameId, double? MeanLogLuminance, double? ClipLow, double? ClipHigh);
|
||||
|
||||
@@ -25,6 +34,12 @@ public sealed class FrameRepository
|
||||
/// <summary>Rows per statement of the batched PostgreSQL updates (4 or fewer parameters per row).</summary>
|
||||
private const int Batch = 1000;
|
||||
|
||||
/// <summary>Rows per multi-row upsert: 30 x 27 parameters stays below SQLite's historical 999-variable limit.</summary>
|
||||
private const int UpsertRows = 30;
|
||||
|
||||
/// <summary>Light statistics belong to the analysis job; a sync from the sidecar never clears them.</summary>
|
||||
private static readonly string[] AnalysisColumns = ["mean_log_luminance", "clip_low", "clip_high"];
|
||||
|
||||
private readonly DbAccess _db;
|
||||
|
||||
internal FrameRepository(DbAccess db) => _db = db;
|
||||
@@ -112,6 +127,89 @@ public sealed class FrameRepository
|
||||
return updated;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Makes the frames of a sequence exactly <paramref name="frames"/> (from its sidecar): rows are upserted by id,
|
||||
/// rows not listed are deleted, light statistics of kept rows survive. Indexes may be renumbered freely.
|
||||
/// </summary>
|
||||
public Task<int> SyncSequenceAsync(Guid sequenceId, IReadOnlyList<Frame> frames, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
// Park current rows on negative indexes so renumbered rows never collide on (sequence_id, idx).
|
||||
await session.ExecuteAsync("UPDATE frames SET idx = -idx - 1 WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
var updates = string.Join(", ", ColumnNames.Skip(1).Where(c => !AnalysisColumns.Contains(c)).Select(c => $"{c} = excluded.{c}"));
|
||||
for (var start = 0; start < frames.Count; start += UpsertRows)
|
||||
{
|
||||
var count = Math.Min(UpsertRows, frames.Count - start);
|
||||
var sql = new StringBuilder("INSERT INTO frames (").Append(Columns).Append(") VALUES ");
|
||||
var args = new DbParam[count * ColumnNames.Length];
|
||||
for (var i = 0; i < count; i++)
|
||||
{
|
||||
var values = Values(frames[start + i]);
|
||||
sql.Append(i > 0 ? ", (" : "(");
|
||||
for (var c = 0; c < values.Length; c++)
|
||||
{
|
||||
args[i * values.Length + c] = new DbParam(values[c]);
|
||||
sql.Append(c > 0 ? ", $" : "$").Append(i * values.Length + c + 1);
|
||||
}
|
||||
sql.Append(')');
|
||||
}
|
||||
sql.Append(" ON CONFLICT (id) DO UPDATE SET ").Append(updates);
|
||||
await session.ExecuteAsync(sql.ToString(), args, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return await session.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1 AND idx < 0", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Number of frames of a sequence.</summary>
|
||||
public async Task<int> CountBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
(int)await _db.ScalarAsync<long>("SELECT count(*) FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
|
||||
/// <summary>Frames of an archive with this size and capture time (card import duplicate candidates).</summary>
|
||||
public Task<IReadOnlyList<Frame>> FindBySizeAndCaptureAsync(Guid archiveId, long size, DateTime capturedAt, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
$"SELECT {string.Join(", ", ColumnNames.Select(c => "f." + c))} FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
|
||||
"WHERE s.archive_id = $1 AND f.size = $2 AND f.captured_at = $3",
|
||||
[archiveId, size, SequenceRepository.Wall(capturedAt)], Map, cancellationToken);
|
||||
|
||||
/// <summary>Every frame of an archive whose SHA-256 occurs more than once, grouped by hash then sequence and index.</summary>
|
||||
public Task<IReadOnlyList<DuplicateFrame>> ListDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
"SELECT f.sha256, f.id, f.sequence_id, f.idx, f.relative_path FROM frames f JOIN sequences s ON s.id = f.sequence_id " +
|
||||
"WHERE s.archive_id = $1 AND f.sha256 IN (SELECT f2.sha256 FROM frames f2 JOIN sequences s2 ON s2.id = f2.sequence_id " +
|
||||
"WHERE s2.archive_id = $1 AND f2.sha256 IS NOT NULL GROUP BY f2.sha256 HAVING count(*) > 1) " +
|
||||
"ORDER BY f.sha256, s.first_capture, f.sequence_id, f.idx",
|
||||
[archiveId], r => new DuplicateFrame(r.GetString(0), r.GetGuid(1), r.GetGuid(2), r.GetInt32(3), r.GetString(4)), cancellationToken);
|
||||
|
||||
/// <summary>Frames of an archive that carry every bit of <paramref name="flags"/>.</summary>
|
||||
public Task<IReadOnlyList<FrameRef>> ListFlaggedAsync(Guid archiveId, FrameFlags flags, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
"SELECT f.id, f.sequence_id FROM frames f JOIN sequences s ON s.id = f.sequence_id WHERE s.archive_id = $1 AND (f.flags & $2) = $2",
|
||||
[archiveId, (int)flags], r => new FrameRef(r.GetGuid(0), r.GetGuid(1)), cancellationToken);
|
||||
|
||||
/// <summary>Rewrites the relative paths of a sequence's frames after its folder moved (mass rename).</summary>
|
||||
/// <remarks>Only paths still under <paramref name="oldFolder"/> change, so repeating it after a crash is harmless.</remarks>
|
||||
public Task<int> RebaseFolderAsync(Guid sequenceId, string oldFolder, string newFolder, CancellationToken cancellationToken = default)
|
||||
{
|
||||
// substr counts characters (code points) on both dialects; the tail starts with the '/' after the folder.
|
||||
var prefix = oldFolder + "/";
|
||||
var length = prefix.EnumerateRunes().Count();
|
||||
return _db.ExecuteAsync(
|
||||
"UPDATE frames SET relative_path = $2 || substr(relative_path, $3) WHERE sequence_id = $1 AND substr(relative_path, 1, $4) = $5",
|
||||
[sequenceId, newFolder, length, length, prefix], cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Writes flags and hashes found by a verify, in one transaction; returns the number of frames updated.</summary>
|
||||
public Task<int> UpdateIntegrityAsync(IReadOnlyList<FrameIntegrity> frames, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var updated = 0;
|
||||
foreach (var f in frames)
|
||||
{
|
||||
updated += await session.ExecuteAsync("UPDATE frames SET flags = $2, sha256 = COALESCE($3, sha256) WHERE id = $1",
|
||||
[f.FrameId, (int)f.Flags, f.Sha256?.ToLowerInvariant()], cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
return updated;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Removes every frame of a sequence; returns how many.</summary>
|
||||
public Task<int> DeleteBySequenceAsync(Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.ExecuteAsync("DELETE FROM frames WHERE sequence_id = $1", [sequenceId], cancellationToken);
|
||||
|
||||
@@ -96,6 +96,16 @@ public sealed class MediaCardRepository
|
||||
await _db.ScalarAsync<int?>("SELECT 1 FROM import_history WHERE card_id = $1 AND path = $2 AND size = $3 AND mtime = $4",
|
||||
[cardId, key.Path, key.Size, key.Modified], cancellationToken).ConfigureAwait(false) is not null;
|
||||
|
||||
/// <summary>Every recorded import from a card that has a hash and a frame (candidates for a verified card erase).</summary>
|
||||
public Task<IReadOnlyList<ImportRecord>> ListVerifiableImportsAsync(Guid cardId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
"SELECT card_id, path, size, mtime, sha256, frame_id, job_id, imported_at FROM import_history " +
|
||||
"WHERE card_id = $1 AND sha256 IS NOT NULL AND frame_id IS NOT NULL ORDER BY path",
|
||||
[cardId],
|
||||
r => new ImportRecord(r.GetGuid(0), new ImportKey(r.GetString(1), r.GetInt64(2), r.GetDateTimeOffset(3)),
|
||||
r.GetStringOrNull(4), r.GetGuidOrNull(5), r.GetGuidOrNull(6), r.GetDateTimeOffset(7)),
|
||||
cancellationToken);
|
||||
|
||||
/// <summary>Most recent imports from a card.</summary>
|
||||
public Task<IReadOnlyList<ImportRecord>> ListImportsAsync(Guid cardId, int limit = 1000, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync(
|
||||
|
||||
@@ -79,6 +79,9 @@ public sealed record ArchiveHealth(
|
||||
long CorruptFrames,
|
||||
long DuplicateFrames);
|
||||
|
||||
/// <summary>Where a sequence lives in its archive.</summary>
|
||||
public sealed record SequenceLocation(Guid Id, string Folder, string? Title);
|
||||
|
||||
/// <summary>Sequences (projects) with their tags; Library listing and Archive statistics.</summary>
|
||||
public sealed class SequenceRepository
|
||||
{
|
||||
@@ -259,6 +262,45 @@ public sealed class SequenceRepository
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Folder and title of every sequence of an archive, by folder.</summary>
|
||||
public Task<IReadOnlyList<SequenceLocation>> ListLocationsAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
||||
_db.ListAsync("SELECT id, folder, title FROM sequences WHERE archive_id = $1 ORDER BY folder, id", [archiveId],
|
||||
r => new SequenceLocation(r.GetGuid(0), r.GetString(1), r.GetStringOrNull(2)), cancellationToken);
|
||||
|
||||
/// <summary>The sequence created for a session key (automatic grouping of uploads and card sessions).</summary>
|
||||
public Task<Guid?> FindBySessionKeyAsync(Guid archiveId, string sessionKey, CancellationToken cancellationToken = default) =>
|
||||
_db.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
|
||||
[archiveId, sessionKey], cancellationToken);
|
||||
|
||||
/// <summary>Binds a session key to a sequence unless it is already bound; returns the sequence the key resolves to.</summary>
|
||||
public Task<Guid> MapSessionKeyAsync(Guid archiveId, string sessionKey, Guid sequenceId, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
await session.ExecuteAsync(
|
||||
"INSERT INTO sequence_sessions (archive_id, session_key, sequence_id) VALUES ($1, $2, $3) ON CONFLICT (archive_id, session_key) DO NOTHING",
|
||||
[archiveId, sessionKey, sequenceId], cancellationToken).ConfigureAwait(false);
|
||||
return (await session.ScalarAsync<Guid?>("SELECT sequence_id FROM sequence_sessions WHERE archive_id = $1 AND session_key = $2",
|
||||
[archiveId, sessionKey], cancellationToken).ConfigureAwait(false))!.Value;
|
||||
}, cancellationToken);
|
||||
|
||||
/// <summary>Moves a sequence to another folder and/or title (mass rename); false when it does not exist.</summary>
|
||||
public Task<bool> RelocateAsync(Guid id, string folder, string? title, CancellationToken cancellationToken = default) =>
|
||||
_db.InTransactionAsync(async session =>
|
||||
{
|
||||
var current = await session.QuerySingleOrDefaultAsync($"{_select} WHERE s.id = $1{_db.Sql.ForUpdate}", [id], Map, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (current is null) return false;
|
||||
var titleChanged = current.Title != title;
|
||||
await UpsertRowAsync(session, current with
|
||||
{
|
||||
Folder = folder,
|
||||
Title = title,
|
||||
SettingsVersion = titleChanged ? current.SettingsVersion + 1 : current.SettingsVersion,
|
||||
UpdatedAt = DateTimeOffset.UtcNow,
|
||||
}, cancellationToken).ConfigureAwait(false);
|
||||
return true;
|
||||
}, cancellationToken);
|
||||
|
||||
private static List<string> Filter(SequenceQuery query, SqlParameters p)
|
||||
{
|
||||
var where = new List<string>();
|
||||
|
||||
Reference in new issue
Block a user