feat(core): card import planning and verified copy engine
Card scanning (DCIM and flat), session splitting with test-shot merging and user splits/merges, per-body import plans named by templates with archive duplicate detection, hash-verified resumable copy with per-file errors, a plain-folder destination and a verified card erase with dry run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
This commit is contained in:
12 files changed
+1252
No files matched your search
@@ -0,0 +1,90 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>A card file that was imported, with the hash recorded at import and where its copy lives.</summary>
|
||||
/// <param name="SourcePath">Path below the card root, forward slashes.</param>
|
||||
/// <param name="CopyPath">Absolute path of the archive (or folder) copy.</param>
|
||||
public sealed record EraseCandidate(string SourcePath, long Size, string Sha256, string CopyPath);
|
||||
|
||||
public enum EraseVerdict
|
||||
{
|
||||
/// <summary>Card file and copy both hash to the recorded SHA-256: safe to delete from the card.</summary>
|
||||
Delete,
|
||||
|
||||
SourceMissing,
|
||||
|
||||
/// <summary>The card file no longer matches what was imported.</summary>
|
||||
SourceChanged,
|
||||
|
||||
CopyMissing,
|
||||
|
||||
/// <summary>The copy no longer matches: the card file is the only good one left.</summary>
|
||||
CopyMismatch,
|
||||
}
|
||||
|
||||
/// <param name="SourceModifiedUtc">Card file time when checked; execution refuses a file touched since.</param>
|
||||
public sealed record EraseItem(EraseCandidate Candidate, EraseVerdict Verdict, DateTime? SourceModifiedUtc);
|
||||
|
||||
/// <summary>The dry run of "delete imported" (BRIEF §12.4, §13): exactly the files that would be deleted, and why
|
||||
/// the others are kept.</summary>
|
||||
public sealed record ErasePlan(string SourceRoot, IReadOnlyList<EraseItem> Items)
|
||||
{
|
||||
public IEnumerable<EraseItem> ToDelete => Items.Where(i => i.Verdict == EraseVerdict.Delete);
|
||||
}
|
||||
|
||||
public sealed record EraseResult(int Deleted, IReadOnlyList<EraseItem> Kept);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes imported photos from a card only after re-hashing both the card file and its copy against the hash
|
||||
/// recorded at import (BRIEF §0.2 rule 8). Never formats (that is the OS dialog) and never touches folders.
|
||||
/// </summary>
|
||||
public static class CardEraser
|
||||
{
|
||||
public static async Task<ErasePlan> PlanAsync(string sourceRoot, IReadOnlyList<EraseCandidate> candidates, IProgress<int>? progress = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = new List<EraseItem>(candidates.Count);
|
||||
for (var i = 0; i < candidates.Count; i++)
|
||||
{
|
||||
var c = candidates[i];
|
||||
var source = new FileInfo(Path.Combine(sourceRoot, c.SourcePath));
|
||||
var verdict = !source.Exists ? EraseVerdict.SourceMissing
|
||||
: !File.Exists(c.CopyPath) ? EraseVerdict.CopyMissing
|
||||
: !await MatchesAsync(c.CopyPath, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.CopyMismatch
|
||||
: !await MatchesAsync(source.FullName, c, cancellationToken).ConfigureAwait(false) ? EraseVerdict.SourceChanged
|
||||
: EraseVerdict.Delete;
|
||||
items.Add(new EraseItem(c, verdict, source.Exists ? source.LastWriteTimeUtc : null));
|
||||
progress?.Report(i + 1);
|
||||
}
|
||||
return new ErasePlan(Path.GetFullPath(sourceRoot), items);
|
||||
}
|
||||
|
||||
/// <summary>Deletes the plan's <see cref="EraseVerdict.Delete"/> files whose size and time are unchanged since planning.</summary>
|
||||
public static EraseResult Execute(ErasePlan plan, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var deleted = 0;
|
||||
var kept = plan.Items.Where(i => i.Verdict != EraseVerdict.Delete).ToList();
|
||||
foreach (var item in plan.ToDelete)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var source = new FileInfo(Path.Combine(plan.SourceRoot, item.Candidate.SourcePath));
|
||||
var copy = new FileInfo(item.Candidate.CopyPath);
|
||||
if (!source.Exists || source.Length != item.Candidate.Size || source.LastWriteTimeUtc != item.SourceModifiedUtc
|
||||
|| !copy.Exists || copy.Length != item.Candidate.Size)
|
||||
{
|
||||
kept.Add(item with { Verdict = EraseVerdict.SourceChanged });
|
||||
continue;
|
||||
}
|
||||
source.Delete();
|
||||
deleted++;
|
||||
Log.Info("import", $"Deleted verified {item.Candidate.SourcePath} from the card");
|
||||
}
|
||||
return new EraseResult(deleted, kept);
|
||||
}
|
||||
|
||||
private static async Task<bool> MatchesAsync(string path, EraseCandidate candidate, CancellationToken cancellationToken) =>
|
||||
new FileInfo(path).Length == candidate.Size
|
||||
&& string.Equals(await ContentHash.Sha256Async(path, null, cancellationToken).ConfigureAwait(false), candidate.Sha256, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Metadata;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>A photo file on a card or source folder.</summary>
|
||||
/// <param name="RelativePath">Path below the source root, forward slashes (the card import history key).</param>
|
||||
public sealed record CardFile(string RelativePath, long Size, DateTime ModifiedUtc);
|
||||
|
||||
/// <summary>A card photo with its light metadata.</summary>
|
||||
public sealed record CardPhoto(CardFile File, FrameMetadata Metadata)
|
||||
{
|
||||
/// <summary>Capture time, or the file time when the photo records none.</summary>
|
||||
public DateTime CaptureTime => Metadata.CapturedAt ?? File.ModifiedUtc;
|
||||
}
|
||||
|
||||
/// <summary>A file that could not be read while planning; it is listed, never a reason to stop.</summary>
|
||||
public sealed record ImportProblem(string RelativePath, string Message);
|
||||
|
||||
/// <summary>Finds photos on a card (BRIEF §12.4, §13): the <c>DCIM</c> tree when present, otherwise the root.</summary>
|
||||
public static class CardScanner
|
||||
{
|
||||
private const int MaxDepth = 12;
|
||||
|
||||
private static readonly EnumerationOptions Listing = new()
|
||||
{
|
||||
IgnoreInaccessible = true,
|
||||
AttributesToSkip = FileAttributes.ReparsePoint | FileAttributes.System | FileAttributes.Hidden,
|
||||
RecurseSubdirectories = false,
|
||||
MatchCasing = MatchCasing.CaseInsensitive,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Photos below <paramref name="root"/>: everything under <c>DCIM</c> (camera folders) when the card has one,
|
||||
/// otherwise the root's photos and, with <paramref name="includeSubfolders"/>, its subfolders'. Ordered by path.
|
||||
/// </summary>
|
||||
public static IReadOnlyList<CardFile> Find(string root, bool includeSubfolders, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var dcim = Directory.EnumerateDirectories(root, "DCIM", Listing).FirstOrDefault();
|
||||
var start = dcim ?? root;
|
||||
var recurse = dcim is not null || includeSubfolders;
|
||||
var files = new List<CardFile>();
|
||||
var pending = new Stack<(string Path, int Depth)>();
|
||||
pending.Push((start, 0));
|
||||
while (pending.Count > 0)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var (directory, depth) = pending.Pop();
|
||||
foreach (var file in new DirectoryInfo(directory).EnumerateFiles("*", Listing))
|
||||
{
|
||||
if (ArchiveScanner.IsPhoto(file.Name))
|
||||
files.Add(new CardFile(Path.GetRelativePath(root, file.FullName).Replace('\\', '/'), file.Length, file.LastWriteTimeUtc));
|
||||
}
|
||||
if (!recurse || depth >= MaxDepth) continue;
|
||||
foreach (var child in Directory.EnumerateDirectories(directory, "*", Listing))
|
||||
{
|
||||
if (!Path.GetFileName(child).StartsWith('.')) pending.Push((child, depth + 1));
|
||||
}
|
||||
}
|
||||
files.Sort((a, b) => string.CompareOrdinal(a.RelativePath, b.RelativePath));
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>Reads light metadata of every file; unreadable files become <paramref name="problems"/>.</summary>
|
||||
public static IReadOnlyList<CardPhoto> ReadMetadata(string root, IReadOnlyList<CardFile> files, List<ImportProblem> problems,
|
||||
IProgress<int>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var photos = new List<CardPhoto>(files.Count);
|
||||
for (var i = 0; i < files.Count; i++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var file = files[i];
|
||||
try
|
||||
{
|
||||
photos.Add(new CardPhoto(file, MetadataReader.Read(Path.Combine(root, file.RelativePath), cancellationToken)));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("import", $"Cannot read {file.RelativePath}", ex);
|
||||
problems.Add(new ImportProblem(file.RelativePath, ex.Message));
|
||||
}
|
||||
progress?.Report(i + 1);
|
||||
}
|
||||
return photos;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
using Titano.Core.Archiving;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>
|
||||
/// Import destination "Folder" (BRIEF §12.4): photos go to <c>root/<folder>/<name></c> without sidecars or index.
|
||||
/// A name already taken by identical content is a duplicate; by other content, the photo gets a suffix. Nothing at the
|
||||
/// destination is ever replaced.
|
||||
/// </summary>
|
||||
public sealed class FolderImportSink(string root) : IImportSink
|
||||
{
|
||||
public string StagingDirectory { get; } = Path.Combine(Path.GetFullPath(root), ArchiveLayout.MetaFolder, ArchiveLayout.StagingFolder);
|
||||
|
||||
public async Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken)
|
||||
{
|
||||
var folder = Path.Combine(Path.GetFullPath(root), session.Folder.Replace('/', Path.DirectorySeparatorChar));
|
||||
Directory.CreateDirectory(folder);
|
||||
var results = new List<SinkCommit>(files.Count);
|
||||
foreach (var file in files)
|
||||
{
|
||||
var name = file.Item.FileName;
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
var duplicate = false;
|
||||
for (var n = 2; File.Exists(Path.Combine(folder, name)); n++)
|
||||
{
|
||||
if (await IsSameAsync(Path.Combine(folder, name), file, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
duplicate = true;
|
||||
break;
|
||||
}
|
||||
name = $"{stem}_{n}{Path.GetExtension(file.Item.FileName)}";
|
||||
}
|
||||
if (!duplicate) File.Move(file.StagedPath, Path.Combine(folder, name), overwrite: false);
|
||||
results.Add(new SinkCommit(null, session.Folder + "/" + name, duplicate));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static async Task<bool> IsSameAsync(string existing, StagedFile file, CancellationToken cancellationToken) =>
|
||||
new FileInfo(existing).Length == file.Item.Size
|
||||
&& await ContentHash.Sha256Async(existing, null, cancellationToken).ConfigureAwait(false) == file.Sha256;
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
using Titano.Core.Diagnostics;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>The card or source folder disappeared mid-import (card removed, share lost). The import can resume.</summary>
|
||||
public sealed class ImportSourceUnavailableException(string root, Exception? inner = null)
|
||||
: IOException(Loc.L("Archive.Error.SourceGone", root), inner)
|
||||
{
|
||||
public string Root { get; } = root;
|
||||
}
|
||||
|
||||
/// <summary>A verified copy waiting in the sink's staging folder.</summary>
|
||||
public sealed record StagedFile(PlannedItem Item, string StagedPath, string Sha256);
|
||||
|
||||
/// <summary>Where a staged file ended up.</summary>
|
||||
/// <param name="FrameId">The archive frame (null for a plain folder destination).</param>
|
||||
/// <param name="Duplicate">The destination already held this content; the staged copy was not used.</param>
|
||||
public sealed record SinkCommit(Guid? FrameId, string RelativePath, bool Duplicate);
|
||||
|
||||
/// <summary>Destination of an import: the archive, a plain folder, or a server (BRIEF §12.4).</summary>
|
||||
public interface IImportSink
|
||||
{
|
||||
/// <summary>Folder for verified copies, on the destination volume so commits are renames.</summary>
|
||||
string StagingDirectory { get; }
|
||||
|
||||
/// <summary>Moves staged files of one session into place; returns one result per file, in order. Files it does
|
||||
/// not move stay where they are and the executor deletes them.</summary>
|
||||
Task<IReadOnlyList<SinkCommit>> CommitAsync(PlannedSession session, IReadOnlyList<StagedFile> files, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public enum ImportItemStatus
|
||||
{
|
||||
Copied,
|
||||
|
||||
/// <summary>Not copied: imported before, or already in the archive.</summary>
|
||||
Skipped,
|
||||
|
||||
/// <summary>Copied and verified, but the destination already held the same content.</summary>
|
||||
Duplicate,
|
||||
|
||||
Failed,
|
||||
}
|
||||
|
||||
public sealed record ImportItemResult(PlannedSession Session, PlannedItem Item, ImportItemStatus Status, string? Sha256, SinkCommit? Commit, string? Error);
|
||||
|
||||
/// <summary>Live figures for the job bar: items and bytes (throughput and ETA come from bytes).</summary>
|
||||
public sealed record ImportProgress(int ItemsDone, int ItemsTotal, long BytesDone, long BytesTotal, int Copied, int Skipped, int Failed, string? Current);
|
||||
|
||||
/// <summary>Results committed since the previous batch; <see cref="NextItem"/> is the resume point once they are recorded.</summary>
|
||||
public sealed record ImportBatch(IReadOnlyList<ImportItemResult> Results, int NextItem, ImportProgress Progress);
|
||||
|
||||
/// <summary>
|
||||
/// Executes an <see cref="ImportPlan"/> (BRIEF §12.4, §13): copies each photo to staging with hashing and read-back
|
||||
/// verification, commits verified copies to the sink in batches, reports per-file errors without stopping, and can
|
||||
/// resume from any batch boundary. When the source disappears it commits what is already verified and throws
|
||||
/// <see cref="ImportSourceUnavailableException"/>.
|
||||
/// </summary>
|
||||
public sealed class ImportExecutor(ImportPlan plan, IImportSink sink)
|
||||
{
|
||||
private const int BatchSize = 32;
|
||||
|
||||
/// <param name="startAt">Number of plan items already done (the last <see cref="ImportBatch.NextItem"/> recorded).</param>
|
||||
/// <param name="committed">Called after each batch; record the results, then checkpoint <see cref="ImportBatch.NextItem"/>.</param>
|
||||
/// <param name="progress">Called before and after each copy (throughput and ETA come from the bytes).</param>
|
||||
public async Task<ImportProgress> RunAsync(int startAt, bool verify, Func<ImportBatch, ValueTask> committed,
|
||||
Func<ImportProgress, ValueTask>? progress = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var items = plan.Sessions.SelectMany(s => s.Items.Select(i => (Session: s, Item: i))).ToList();
|
||||
var totalBytes = items.Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size);
|
||||
var bytesDone = items.Take(startAt).Where(x => x.Item.Action == ImportAction.Copy).Sum(x => x.Item.Size);
|
||||
int copied = 0, skipped = 0, failed = 0;
|
||||
var results = new List<ImportItemResult>();
|
||||
var staged = new List<StagedFile>();
|
||||
PlannedSession? stagedSession = null;
|
||||
var next = startAt;
|
||||
|
||||
ImportProgress Snapshot(string? current) => new(next, items.Count, bytesDone, totalBytes, copied, skipped, failed, current);
|
||||
|
||||
async Task FlushAsync()
|
||||
{
|
||||
if (staged.Count > 0)
|
||||
{
|
||||
var commits = await sink.CommitAsync(stagedSession!, staged, cancellationToken).ConfigureAwait(false);
|
||||
for (var i = 0; i < staged.Count; i++)
|
||||
{
|
||||
var duplicate = commits[i].Duplicate;
|
||||
results.Add(new ImportItemResult(stagedSession!, staged[i].Item, duplicate ? ImportItemStatus.Duplicate : ImportItemStatus.Copied,
|
||||
staged[i].Sha256, commits[i], null));
|
||||
if (duplicate) skipped++;
|
||||
else copied++;
|
||||
DeleteLeftover(staged[i].StagedPath);
|
||||
}
|
||||
staged.Clear();
|
||||
stagedSession = null;
|
||||
}
|
||||
if (results.Count == 0) return;
|
||||
await committed(new ImportBatch([.. results], next, Snapshot(null))).ConfigureAwait(false);
|
||||
results.Clear();
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(sink.StagingDirectory);
|
||||
for (; next < items.Count; next++)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var (session, item) = items[next];
|
||||
if ((stagedSession is not null && !ReferenceEquals(stagedSession, session)) || staged.Count + results.Count >= BatchSize)
|
||||
await FlushAsync().ConfigureAwait(false);
|
||||
if (item.Action != ImportAction.Copy)
|
||||
{
|
||||
skipped++;
|
||||
results.Add(new ImportItemResult(session, item, ImportItemStatus.Skipped, item.Sha256, null, null));
|
||||
continue;
|
||||
}
|
||||
var source = Path.Combine(plan.SourceRoot, item.SourcePath);
|
||||
if (progress is not null) await progress(Snapshot(item.SourcePath)).ConfigureAwait(false);
|
||||
var target = Path.Combine(sink.StagingDirectory, $"{plan.Id:N}-{next:D6}{Path.GetExtension(item.FileName)}");
|
||||
DeleteLeftover(target);
|
||||
try
|
||||
{
|
||||
if (new FileInfo(source) is not { Exists: true } info || info.Length != item.Size)
|
||||
throw new IOException(Loc.L("Archive.Error.SourceChanged", item.SourcePath));
|
||||
var sha = await VerifiedCopy.CopyAsync(source, target, verify, read => bytesDone += read, cancellationToken).ConfigureAwait(false);
|
||||
staged.Add(new StagedFile(item, target, sha));
|
||||
stagedSession = session;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
if (!Directory.Exists(plan.SourceRoot))
|
||||
{
|
||||
await FlushAsync().ConfigureAwait(false);
|
||||
throw new ImportSourceUnavailableException(plan.SourceRoot, ex);
|
||||
}
|
||||
Log.Warning("import", $"Copy of {item.SourcePath} failed", ex);
|
||||
failed++;
|
||||
results.Add(new ImportItemResult(session, item, ImportItemStatus.Failed, null, null, ex.Message));
|
||||
}
|
||||
}
|
||||
await FlushAsync().ConfigureAwait(false);
|
||||
return Snapshot(null);
|
||||
}
|
||||
|
||||
private static void DeleteLeftover(string staged)
|
||||
{
|
||||
// Staged files are this import's own copies of card files; the originals stay on the card.
|
||||
if (File.Exists(staged)) File.Delete(staged);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Titano.Core.Imaging;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>What to do with a photo that is already in the archive (same SHA-256).</summary>
|
||||
public enum DuplicatePolicy
|
||||
{
|
||||
/// <summary>Leave it on the card; the archive copy is identical.</summary>
|
||||
Skip,
|
||||
|
||||
/// <summary>Import it again into this session's sequence (its name gets a suffix if taken).</summary>
|
||||
ImportAgain,
|
||||
}
|
||||
|
||||
/// <summary>Card import settings (BRIEF §12.4, §13); the defaults are the brief's.</summary>
|
||||
public sealed record ImportOptions
|
||||
{
|
||||
public string FolderTemplate { get; init; } = TemplateDefaults.Folder;
|
||||
public string FileTemplate { get; init; } = TemplateDefaults.File;
|
||||
public TimeSpan SessionGap { get; init; } = TimeSpan.FromMinutes(45);
|
||||
public int MinimumSessionFrames { get; init; } = 8;
|
||||
public bool IncludeSubfolders { get; init; } = true;
|
||||
public bool VerifyCopies { get; init; } = true;
|
||||
public DuplicatePolicy Duplicates { get; init; } = DuplicatePolicy.Skip;
|
||||
|
||||
/// <summary>Card paths where the user split a session (a new session starts at that photo).</summary>
|
||||
public IReadOnlySet<string> SplitBefore { get; init; } = new HashSet<string>();
|
||||
|
||||
/// <summary>Card paths where the user merged a session into the previous one.</summary>
|
||||
public IReadOnlySet<string> MergeAt { get; init; } = new HashSet<string>();
|
||||
|
||||
/// <summary>Titles given to sessions, by session key; used as the sequence title and the <c>{title}</c> token.</summary>
|
||||
public IReadOnlyDictionary<string, string> Titles { get; init; } = new Dictionary<string, string>();
|
||||
}
|
||||
|
||||
public enum ImportAction
|
||||
{
|
||||
Copy,
|
||||
|
||||
/// <summary>Imported from this card before (same path, size and time).</summary>
|
||||
SkipImported,
|
||||
|
||||
/// <summary>Already in the archive (same size and capture time, then the same SHA-256).</summary>
|
||||
SkipInArchive,
|
||||
}
|
||||
|
||||
/// <summary>One photo of the plan and its destination name.</summary>
|
||||
public sealed record PlannedItem
|
||||
{
|
||||
/// <summary>Path below the source root, forward slashes.</summary>
|
||||
public required string SourcePath { get; init; }
|
||||
|
||||
public long Size { get; init; }
|
||||
public DateTime ModifiedUtc { get; init; }
|
||||
public DateTime CapturedAt { get; init; }
|
||||
public required string FileName { get; init; }
|
||||
public ImportAction Action { get; init; }
|
||||
|
||||
/// <summary>The archive frame this photo duplicates (<see cref="ImportAction.SkipInArchive"/>, or imported again).</summary>
|
||||
public Guid? ExistingFrameId { get; init; }
|
||||
|
||||
/// <summary>SHA-256 computed while planning (duplicate check), if any.</summary>
|
||||
public string? Sha256 { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>A session of the plan: one camera body and file format, one sequence in the archive.</summary>
|
||||
public sealed record PlannedSession
|
||||
{
|
||||
/// <summary>1-based, in order of start time across the import.</summary>
|
||||
public int Number { get; init; }
|
||||
|
||||
/// <summary>Stable key of the session; re-planning the same photos yields the same key, so an interrupted import
|
||||
/// resumes into the same sequence.</summary>
|
||||
public required string SessionKey { get; init; }
|
||||
|
||||
public string? CameraMake { get; init; }
|
||||
public string? CameraModel { get; init; }
|
||||
public string? CameraSerial { get; init; }
|
||||
public ImageFileFormat Format { get; init; }
|
||||
public DateTime Start { get; init; }
|
||||
public DateTime End { get; init; }
|
||||
|
||||
/// <summary>Destination folder relative to the destination root, forward slashes.</summary>
|
||||
public required string Folder { get; init; }
|
||||
|
||||
public string? Title { get; init; }
|
||||
public IReadOnlyList<PlannedItem> Items { get; init; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The exact result of an import before any byte is written (BRIEF §12.4 "Plan before touching disk"): sessions,
|
||||
/// folders, file names, what is skipped and why, and the space it needs. Executing it copies exactly this.
|
||||
/// </summary>
|
||||
public sealed record ImportPlan
|
||||
{
|
||||
public required Guid Id { get; init; }
|
||||
public required string SourceRoot { get; init; }
|
||||
public required string DestinationRoot { get; init; }
|
||||
public DateTimeOffset Created { get; init; }
|
||||
public IReadOnlyList<PlannedSession> Sessions { get; init; } = [];
|
||||
public IReadOnlyList<ImportProblem> Problems { get; init; } = [];
|
||||
|
||||
/// <summary>Free space at the destination when planned; null when the volume does not report it.</summary>
|
||||
public long? AvailableBytes { get; init; }
|
||||
|
||||
[JsonIgnore]
|
||||
public IEnumerable<PlannedItem> Items => Sessions.SelectMany(s => s.Items);
|
||||
[JsonIgnore]
|
||||
public int FileCount => Sessions.Sum(s => s.Items.Count);
|
||||
[JsonIgnore]
|
||||
public int CopyCount => Items.Count(i => i.Action == ImportAction.Copy);
|
||||
[JsonIgnore]
|
||||
public long CopyBytes => Items.Where(i => i.Action == ImportAction.Copy).Sum(i => i.Size);
|
||||
[JsonIgnore]
|
||||
public bool HasEnoughSpace => AvailableBytes is not { } free || free >= CopyBytes;
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Naming;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>An archive frame that may be the same photo.</summary>
|
||||
public sealed record ArchiveMatch(Guid FrameId, string? Sha256);
|
||||
|
||||
/// <summary>What the planner needs to know about the destination (implemented over the archive index).</summary>
|
||||
public interface IImportLookup
|
||||
{
|
||||
/// <summary>The file was imported from this card before (same path, size and time).</summary>
|
||||
Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Archive frames with this size and capture time; the planner compares hashes only for these.</summary>
|
||||
Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Folder of the sequence an earlier import of this session went to, so a re-import continues it.</summary>
|
||||
Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>The folder is taken by something else (another sequence), so the session needs another name.</summary>
|
||||
bool IsFolderTaken(string folder);
|
||||
}
|
||||
|
||||
/// <summary>Turns card photos into an <see cref="ImportPlan"/> (BRIEF §12.4, §13). Nothing is written.</summary>
|
||||
public static class ImportPlanner
|
||||
{
|
||||
public static async Task<ImportPlan> PlanAsync(string sourceRoot, string destinationRoot, IReadOnlyList<CardPhoto> photos,
|
||||
IReadOnlyList<ImportProblem> problems, ImportOptions options, IImportLookup lookup, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var folderTemplate = PathTemplate.Parse(options.FolderTemplate);
|
||||
var fileTemplate = PathTemplate.Parse(options.FileTemplate);
|
||||
var drafts = new List<(string Key, List<CardPhoto> Photos)>();
|
||||
foreach (var body in photos.GroupBy(BodyKey, StringComparer.Ordinal))
|
||||
{
|
||||
var ordered = body.OrderBy(p => p.CaptureTime).ThenBy(p => p.File.RelativePath, StringComparer.Ordinal).ToList();
|
||||
var ranges = SessionSplitter.Split(ordered.ConvertAll(p => p.CaptureTime), options.SessionGap, options.MinimumSessionFrames,
|
||||
i => options.SplitBefore.Contains(ordered[i].File.RelativePath) ? true : options.MergeAt.Contains(ordered[i].File.RelativePath) ? false : null);
|
||||
foreach (var range in ranges)
|
||||
{
|
||||
var members = ordered.GetRange(range.Start, range.Count);
|
||||
drafts.Add((SessionKey(body.Key, members[0].CaptureTime), members));
|
||||
}
|
||||
}
|
||||
drafts.Sort((a, b) => a.Photos[0].CaptureTime.CompareTo(b.Photos[0].CaptureTime) is var c and not 0 ? c : string.CompareOrdinal(a.Key, b.Key));
|
||||
|
||||
var claimedFolders = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var sessions = new List<PlannedSession>(drafts.Count);
|
||||
for (var s = 0; s < drafts.Count; s++)
|
||||
{
|
||||
var (key, members) = drafts[s];
|
||||
var first = members[0];
|
||||
options.Titles.TryGetValue(key, out var title);
|
||||
var context = Context(first, first.CaptureTime, s + 1, 1, title);
|
||||
var folder = await lookup.FindSessionFolderAsync(key, cancellationToken).ConfigureAwait(false)
|
||||
?? UniqueFolder(folderTemplate.ExpandFolder(context), claimedFolders, lookup);
|
||||
claimedFolders.Add(folder);
|
||||
|
||||
var names = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var items = new List<PlannedItem>(members.Count);
|
||||
for (var i = 0; i < members.Count; i++)
|
||||
{
|
||||
var photo = members[i];
|
||||
var name = UniqueName(fileTemplate.ExpandFile(Context(photo, first.CaptureTime, s + 1, i + 1, title)), names);
|
||||
items.Add(await PlanItemAsync(sourceRoot, photo, name, options, lookup, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
sessions.Add(new PlannedSession
|
||||
{
|
||||
Number = s + 1,
|
||||
SessionKey = key,
|
||||
CameraMake = first.Metadata.Make,
|
||||
CameraModel = first.Metadata.Model,
|
||||
CameraSerial = first.Metadata.BodySerial,
|
||||
Format = first.Metadata.Format,
|
||||
Start = first.CaptureTime,
|
||||
End = members[^1].CaptureTime,
|
||||
Folder = folder,
|
||||
Title = title,
|
||||
Items = items,
|
||||
});
|
||||
}
|
||||
return new ImportPlan
|
||||
{
|
||||
Id = Guid.CreateVersion7(),
|
||||
SourceRoot = Path.GetFullPath(sourceRoot),
|
||||
DestinationRoot = Path.GetFullPath(destinationRoot),
|
||||
Created = DateTimeOffset.UtcNow,
|
||||
Sessions = sessions,
|
||||
Problems = problems,
|
||||
AvailableBytes = FreeSpace(destinationRoot),
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<PlannedItem> PlanItemAsync(string sourceRoot, CardPhoto photo, string name, ImportOptions options, IImportLookup lookup,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var item = new PlannedItem
|
||||
{
|
||||
SourcePath = photo.File.RelativePath,
|
||||
Size = photo.File.Size,
|
||||
ModifiedUtc = photo.File.ModifiedUtc,
|
||||
CapturedAt = photo.CaptureTime,
|
||||
FileName = name,
|
||||
};
|
||||
if (await lookup.WasImportedAsync(photo.File, cancellationToken).ConfigureAwait(false))
|
||||
return item with { Action = ImportAction.SkipImported };
|
||||
var candidates = await lookup.FindCandidatesAsync(photo.File.Size, photo.CaptureTime, cancellationToken).ConfigureAwait(false);
|
||||
if (candidates.Count == 0) return item;
|
||||
var sha = await ContentHash.Sha256Async(Path.Combine(sourceRoot, photo.File.RelativePath), null, cancellationToken).ConfigureAwait(false);
|
||||
var match = candidates.FirstOrDefault(c => sha.Equals(c.Sha256, StringComparison.OrdinalIgnoreCase));
|
||||
if (match is null) return item with { Sha256 = sha };
|
||||
return item with
|
||||
{
|
||||
Sha256 = sha,
|
||||
ExistingFrameId = match.FrameId,
|
||||
Action = options.Duplicates == DuplicatePolicy.Skip ? ImportAction.SkipInArchive : ImportAction.Copy,
|
||||
};
|
||||
}
|
||||
|
||||
private static NamingContext Context(CardPhoto photo, DateTime sessionStart, int session, int number, string? title) => new()
|
||||
{
|
||||
CapturedAt = photo.CaptureTime,
|
||||
SessionStart = sessionStart,
|
||||
Session = session,
|
||||
Number = number,
|
||||
NumberInSession = number,
|
||||
CameraMake = photo.Metadata.Make,
|
||||
CameraModel = photo.Metadata.Model,
|
||||
Lens = photo.Metadata.LensModel,
|
||||
Iso = photo.Metadata.Iso,
|
||||
ExposureTime = photo.Metadata.ExposureTime,
|
||||
FNumber = photo.Metadata.FNumber,
|
||||
OriginalName = Path.GetFileNameWithoutExtension(photo.File.RelativePath),
|
||||
Extension = Path.GetExtension(photo.File.RelativePath).TrimStart('.'),
|
||||
Title = title,
|
||||
};
|
||||
|
||||
/// <summary>Camera body and file format: RAW+JPEG shooting yields two sequences instead of doubled frames.</summary>
|
||||
private static string BodyKey(CardPhoto p) => $"{p.Metadata.Make}|{p.Metadata.Model}|{p.Metadata.BodySerial}|{p.Metadata.Format}";
|
||||
|
||||
private static string SessionKey(string bodyKey, DateTime start) =>
|
||||
"card-" + Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(bodyKey)))[..8] + "-"
|
||||
+ start.ToString("yyyyMMdd-HHmmss", CultureInfo.InvariantCulture);
|
||||
|
||||
private static string UniqueFolder(string folder, HashSet<string> claimed, IImportLookup lookup)
|
||||
{
|
||||
if (folder.Length == 0) folder = "Import";
|
||||
var candidate = folder;
|
||||
for (var n = 2; claimed.Contains(candidate) || lookup.IsFolderTaken(candidate); n++)
|
||||
candidate = string.Create(CultureInfo.InvariantCulture, $"{folder} ({n})");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/// <summary>Adds _2, _3… before the extension when the template gives two photos the same name.</summary>
|
||||
internal static string UniqueName(string name, HashSet<string> taken)
|
||||
{
|
||||
var candidate = name;
|
||||
var stem = Path.GetFileNameWithoutExtension(name);
|
||||
var extension = Path.GetExtension(name);
|
||||
for (var n = 2; !taken.Add(candidate); n++) candidate = string.Create(CultureInfo.InvariantCulture, $"{stem}_{n}{extension}");
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private static long? FreeSpace(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var full = Path.GetFullPath(path);
|
||||
while (!Directory.Exists(full) && Path.GetDirectoryName(full) is { } parent) full = parent;
|
||||
// Windows wants a drive root; Unix accepts any directory and reports the volume it is mounted on.
|
||||
return new DriveInfo(OperatingSystem.IsWindows() ? Path.GetPathRoot(full)! : full).AvailableFreeSpace;
|
||||
}
|
||||
catch (Exception ex) when (ex is ArgumentException or IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>A run of consecutive frames forming one session: <c>[Start, Start + Count)</c>.</summary>
|
||||
public readonly record struct SessionRange(int Start, int Count)
|
||||
{
|
||||
public int End => Start + Count;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Splits a time-ordered run of captures into shooting sessions (BRIEF §13): a pause longer than the gap starts a
|
||||
/// new session; a session with fewer than the minimum frames (test shots) merges into the previous one, or into the
|
||||
/// next when it comes first. Breaks the user forced are never merged away; breaks the user removed never split.
|
||||
/// </summary>
|
||||
public static class SessionSplitter
|
||||
{
|
||||
/// <param name="times">Capture times in ascending order.</param>
|
||||
/// <param name="userBreak">For a frame index: true = the user starts a session there, false = the user joined it
|
||||
/// to the previous frame, null = automatic.</param>
|
||||
public static IReadOnlyList<SessionRange> Split(IReadOnlyList<DateTime> times, TimeSpan gap, int minimumFrames, Func<int, bool?>? userBreak = null)
|
||||
{
|
||||
if (times.Count == 0) return [];
|
||||
var groups = new List<(int Start, int Count, bool Forced)>();
|
||||
var start = 0;
|
||||
var forced = false;
|
||||
for (var i = 1; i < times.Count; i++)
|
||||
{
|
||||
var decision = userBreak?.Invoke(i);
|
||||
if (decision == false || (decision is null && times[i] - times[i - 1] <= gap)) continue;
|
||||
groups.Add((start, i - start, forced));
|
||||
start = i;
|
||||
forced = decision == true;
|
||||
}
|
||||
groups.Add((start, times.Count - start, forced));
|
||||
|
||||
for (var i = 0; i < groups.Count && groups.Count > 1;)
|
||||
{
|
||||
var g = groups[i];
|
||||
if (g.Count >= minimumFrames)
|
||||
{
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (i > 0 && !g.Forced)
|
||||
{
|
||||
var previous = groups[i - 1];
|
||||
groups[i - 1] = (previous.Start, previous.Count + g.Count, previous.Forced);
|
||||
groups.RemoveAt(i);
|
||||
}
|
||||
else if (i == 0 && !groups[1].Forced)
|
||||
{
|
||||
groups[1] = (g.Start, g.Count + groups[1].Count, g.Forced);
|
||||
groups.RemoveAt(0);
|
||||
}
|
||||
else
|
||||
{
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return groups.ConvertAll(g => new SessionRange(g.Start, g.Count));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
using System.Security.Cryptography;
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Localization;
|
||||
|
||||
namespace Titano.Core.Import;
|
||||
|
||||
/// <summary>The copy read back from the destination does not match what was read from the source.</summary>
|
||||
public sealed class CopyVerificationException(string path)
|
||||
: IOException(Loc.L("Archive.Error.CopyMismatch", Path.GetFileName(path)))
|
||||
{
|
||||
public string FilePath { get; } = path;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies a file while hashing it, flushes the copy to disk, then re-reads it and compares hashes (BRIEF §12.4).
|
||||
/// A failed or interrupted copy never leaves a partial file behind.
|
||||
/// </summary>
|
||||
public static class VerifiedCopy
|
||||
{
|
||||
/// <summary>Copies <paramref name="source"/> to the new file <paramref name="destination"/>; returns the SHA-256.</summary>
|
||||
/// <param name="verify">Re-read the copy and compare (the default import setting).</param>
|
||||
/// <param name="bytesCopied">Receives the bytes of each chunk read from the source.</param>
|
||||
/// <exception cref="CopyVerificationException">The re-read copy differs.</exception>
|
||||
public static async Task<string> CopyAsync(string source, string destination, bool verify, Action<long>? bytesCopied = null,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(destination))!);
|
||||
var created = false;
|
||||
try
|
||||
{
|
||||
string sha;
|
||||
DateTime modified;
|
||||
await using (var input = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read, ContentHash.BufferSize,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
await using (var output = new FileStream(destination, FileMode.CreateNew, FileAccess.Write, FileShare.None, ContentHash.BufferSize,
|
||||
FileOptions.Asynchronous))
|
||||
{
|
||||
created = true;
|
||||
modified = File.GetLastWriteTimeUtc(source);
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
var buffer = new byte[ContentHash.BufferSize];
|
||||
int read;
|
||||
while ((read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await output.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
|
||||
bytesCopied?.Invoke(read);
|
||||
}
|
||||
await output.FlushAsync(cancellationToken).ConfigureAwait(false);
|
||||
output.Flush(flushToDisk: true);
|
||||
sha = Convert.ToHexStringLower(hash.GetHashAndReset());
|
||||
}
|
||||
File.SetLastWriteTimeUtc(destination, modified);
|
||||
if (verify && await ContentHash.Sha256Async(destination, null, cancellationToken).ConfigureAwait(false) != sha)
|
||||
throw new CopyVerificationException(destination);
|
||||
return sha;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Only our own partial or unverified copy; an existing file at the destination is never touched.
|
||||
if (created && File.Exists(destination)) File.Delete(destination);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Tests.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Core.Tests.Import.ImportTestKit;
|
||||
|
||||
namespace Titano.Core.Tests.Import;
|
||||
|
||||
[Category("import")]
|
||||
public sealed class ImportExecutorTests(TestContext context)
|
||||
{
|
||||
private static readonly TimeSpan Interval = TimeSpan.FromSeconds(10);
|
||||
|
||||
private string Card(int frames = 70)
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
TestPhotos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, frames, Interval);
|
||||
return card;
|
||||
}
|
||||
|
||||
private static Task<ImportProgress> RunAsync(ImportPlan plan, IImportSink sink, int startAt = 0, Func<ImportBatch, ValueTask>? committed = null,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
new ImportExecutor(plan, sink).RunAsync(startAt, verify: true, committed ?? (_ => ValueTask.CompletedTask), null, cancellationToken);
|
||||
|
||||
[Test]
|
||||
public async Task VerifiedCopyHashesPreservesTheTimeAndNeverOverwrites()
|
||||
{
|
||||
var dir = context.CreateTempDirectory();
|
||||
var source = TestPhotos.Write(Path.Combine(dir, "a.jpg"), Start, 1);
|
||||
File.SetLastWriteTimeUtc(source, new DateTime(2026, 9, 28, 16, 0, 0, DateTimeKind.Utc));
|
||||
var target = Path.Combine(dir, "copy", "a.jpg");
|
||||
var sha = await VerifiedCopy.CopyAsync(source, target, verify: true);
|
||||
Assert.Equal(await ContentHash.Sha256Async(source), sha);
|
||||
Assert.BytesEqual(File.ReadAllBytes(source), File.ReadAllBytes(target));
|
||||
Assert.Equal(File.GetLastWriteTimeUtc(source), File.GetLastWriteTimeUtc(target));
|
||||
|
||||
var other = TestPhotos.Write(Path.Combine(dir, "b.jpg"), Start, 2);
|
||||
await Assert.ThrowsAsync<IOException>(() => VerifiedCopy.CopyAsync(other, target, verify: true));
|
||||
Assert.BytesEqual(File.ReadAllBytes(source), File.ReadAllBytes(target));
|
||||
await Assert.ThrowsAsync<FileNotFoundException>(() => VerifiedCopy.CopyAsync(Path.Combine(dir, "gone.jpg"), Path.Combine(dir, "c.jpg"), true));
|
||||
Assert.FileMissing(Path.Combine(dir, "c.jpg"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APlanIsCopiedExactlyIntoAFolder()
|
||||
{
|
||||
var card = Card();
|
||||
var destination = context.CreateTempDirectory("dest");
|
||||
var plan = await PlanAsync(card, destination);
|
||||
var batches = new List<ImportBatch>();
|
||||
var final = await RunAsync(plan, new FolderImportSink(destination), committed: b =>
|
||||
{
|
||||
batches.Add(b);
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
Assert.Equal(70, final.Copied);
|
||||
Assert.Equal(plan.CopyBytes, final.BytesDone);
|
||||
Assert.SequenceEqual(plan.Items.Select(i => plan.Sessions[0].Folder + "/" + i.FileName).Order(StringComparer.Ordinal), Files(destination));
|
||||
Assert.Equal(70, batches[^1].NextItem);
|
||||
Assert.True(batches.Count >= 3);
|
||||
Assert.Empty(Directory.EnumerateFiles(new FolderImportSink(destination).StagingDirectory));
|
||||
foreach (var item in plan.Items)
|
||||
{
|
||||
Assert.BytesEqual(File.ReadAllBytes(Path.Combine(card, item.SourcePath)),
|
||||
File.ReadAllBytes(Path.Combine(destination, plan.Sessions[0].Folder, item.FileName)));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnInterruptedImportResumesFromItsLastBatch()
|
||||
{
|
||||
var card = Card();
|
||||
var destination = context.CreateTempDirectory("dest");
|
||||
var plan = await PlanAsync(card, destination);
|
||||
using var cts = new CancellationTokenSource();
|
||||
var checkpoint = 0;
|
||||
await Assert.ThrowsAsync<OperationCanceledException>(() => RunAsync(plan, new FolderImportSink(destination), committed: b =>
|
||||
{
|
||||
checkpoint = b.NextItem;
|
||||
cts.Cancel();
|
||||
return ValueTask.CompletedTask;
|
||||
}, cancellationToken: cts.Token));
|
||||
Assert.True(checkpoint is > 0 and < 70);
|
||||
var final = await RunAsync(plan, new FolderImportSink(destination), startAt: checkpoint);
|
||||
Assert.Equal(70 - checkpoint, final.Copied);
|
||||
|
||||
var reference = context.CreateTempDirectory("reference");
|
||||
await RunAsync(await PlanAsync(card, reference), new FolderImportSink(reference));
|
||||
Assert.SequenceEqual(Files(reference), Files(destination));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RepeatingCommittedItemsYieldsDuplicatesNotSecondCopies()
|
||||
{
|
||||
var card = Card(20);
|
||||
var destination = context.CreateTempDirectory("dest");
|
||||
var plan = await PlanAsync(card, destination);
|
||||
await RunAsync(plan, new FolderImportSink(destination));
|
||||
var again = await RunAsync(plan, new FolderImportSink(destination), startAt: 10);
|
||||
Assert.Equal(0, again.Copied);
|
||||
Assert.Equal(10, again.Skipped);
|
||||
Assert.Equal(20, Files(destination).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ARemovedCardStopsTheImportWhichResumesOnReinsertion()
|
||||
{
|
||||
var card = Card();
|
||||
var destination = context.CreateTempDirectory("dest");
|
||||
var plan = await PlanAsync(card, destination);
|
||||
var aside = card + "-removed";
|
||||
var checkpoint = 0;
|
||||
await Assert.ThrowsAsync<ImportSourceUnavailableException>(() => RunAsync(plan, new FolderImportSink(destination), committed: b =>
|
||||
{
|
||||
checkpoint = b.NextItem;
|
||||
if (Directory.Exists(card)) Directory.Move(card, aside); // the card is pulled after the first batch
|
||||
return ValueTask.CompletedTask;
|
||||
}));
|
||||
Assert.Equal(checkpoint, Files(destination).Count);
|
||||
Assert.Empty(Directory.EnumerateFiles(new FolderImportSink(destination).StagingDirectory));
|
||||
|
||||
Directory.Move(aside, card);
|
||||
var final = await RunAsync(plan, new FolderImportSink(destination), startAt: checkpoint);
|
||||
Assert.Equal(70 - checkpoint, final.Copied);
|
||||
Assert.Equal(70, Files(destination).Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFileThatChangedSinceThePlanFailsAloneAndTheRestIsCopied()
|
||||
{
|
||||
var card = Card(20);
|
||||
var destination = context.CreateTempDirectory("dest");
|
||||
var plan = await PlanAsync(card, destination);
|
||||
File.AppendAllText(Path.Combine(card, plan.Sessions[0].Items[5].SourcePath), "edited");
|
||||
var failed = new List<ImportItemResult>();
|
||||
var final = await RunAsync(plan, new FolderImportSink(destination), committed: b =>
|
||||
{
|
||||
failed.AddRange(b.Results.Where(r => r.Status == ImportItemStatus.Failed));
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
Assert.Equal(19, final.Copied);
|
||||
Assert.Equal(1, final.Failed);
|
||||
Assert.Equal(plan.Sessions[0].Items[5].SourcePath, failed.Single().Item.SourcePath);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ErasingDeletesOnlyFilesWhoseCardAndCopyStillMatch()
|
||||
{
|
||||
var card = Card(6);
|
||||
var destination = context.CreateTempDirectory("dest");
|
||||
var plan = await PlanAsync(card, destination, new ImportOptions { MinimumSessionFrames = 1 });
|
||||
var results = new List<ImportItemResult>();
|
||||
await RunAsync(plan, new FolderImportSink(destination), committed: b =>
|
||||
{
|
||||
results.AddRange(b.Results);
|
||||
return ValueTask.CompletedTask;
|
||||
});
|
||||
var candidates = results.Select(r => new EraseCandidate(r.Item.SourcePath, r.Item.Size, r.Sha256!, Path.Combine(destination, r.Commit!.RelativePath)))
|
||||
.ToList();
|
||||
// Copy of item 1 damaged after import, card file of item 2 edited, copy of item 3 deleted.
|
||||
var damaged = File.ReadAllBytes(candidates[1].CopyPath);
|
||||
damaged[^10] ^= 0xFF;
|
||||
File.WriteAllBytes(candidates[1].CopyPath, damaged);
|
||||
File.AppendAllText(Path.Combine(card, candidates[2].SourcePath), "x");
|
||||
File.Delete(candidates[3].CopyPath);
|
||||
|
||||
var erase = await CardEraser.PlanAsync(card, candidates);
|
||||
Assert.SequenceEqual(
|
||||
[EraseVerdict.Delete, EraseVerdict.CopyMismatch, EraseVerdict.SourceChanged, EraseVerdict.CopyMissing, EraseVerdict.Delete, EraseVerdict.Delete],
|
||||
erase.Items.Select(i => i.Verdict));
|
||||
Assert.Equal(6, Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Count());
|
||||
|
||||
var done = CardEraser.Execute(erase);
|
||||
Assert.Equal(3, done.Deleted);
|
||||
Assert.Equal(3, done.Kept.Count);
|
||||
var left = Directory.EnumerateFiles(card, "*.JPG", SearchOption.AllDirectories).Select(f => Path.GetRelativePath(card, f).Replace('\\', '/')).Order();
|
||||
Assert.SequenceEqual(candidates.Skip(1).Take(3).Select(c => c.SourcePath).Order(), left);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
using Titano.Core.Archiving;
|
||||
using Titano.Core.Import;
|
||||
using Titano.Core.Tests.Archiving;
|
||||
using Titano.Testing;
|
||||
using static Titano.Core.Tests.Import.ImportTestKit;
|
||||
|
||||
namespace Titano.Core.Tests.Import;
|
||||
|
||||
[Category("import")]
|
||||
public sealed class ImportPlannerTests(TestContext context)
|
||||
{
|
||||
private static readonly TimeSpan Interval = TimeSpan.FromSeconds(10);
|
||||
|
||||
[Test]
|
||||
public async Task ADcimCardIsPlannedWithTheDefaultTemplates()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
TestPhotos.WriteSeries(Path.Combine(card, "DCIM", "100CANON"), Start, 10, Interval);
|
||||
File.WriteAllBytes(Path.Combine(card, "DCIM", "100CANON", "._IMG_0001.JPG"), [0, 5, 22]);
|
||||
File.WriteAllText(Path.Combine(card, "DCIM", "100CANON", "notes.txt"), "x");
|
||||
TestPhotos.Write(Path.Combine(card, "MISC", "IMG_9999.JPG"), Start, 99);
|
||||
|
||||
var plan = await PlanAsync(card, context.CreateTempDirectory("archive"));
|
||||
var session = plan.Sessions.Single();
|
||||
Assert.Equal("2026/2026-09-28 Canon EOS R5", session.Folder);
|
||||
Assert.Equal(10, plan.FileCount);
|
||||
Assert.Equal(10, plan.CopyCount);
|
||||
Assert.SequenceEqual(
|
||||
["2026-09-28_18-00-00_0001.JPG", "2026-09-28_18-00-10_0002.JPG", "2026-09-28_18-00-20_0003.JPG"],
|
||||
session.Items.Take(3).Select(i => i.FileName));
|
||||
Assert.Equal("DCIM/100CANON/IMG_0001.JPG", session.Items[0].SourcePath);
|
||||
Assert.Equal(plan.Items.Sum(i => i.Size), plan.CopyBytes);
|
||||
Assert.NotNull(plan.AvailableBytes);
|
||||
Assert.True(plan.HasEnoughSpace);
|
||||
Assert.Empty(plan.Problems);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFlatFolderIncludesSubfoldersOnlyWhenAsked()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
TestPhotos.WriteSeries(card, Start, 8, Interval);
|
||||
TestPhotos.WriteSeries(Path.Combine(card, "more"), Start.AddSeconds(80), 8, Interval, firstSeed: 50, firstNumber: 9);
|
||||
var destination = context.CreateTempDirectory("archive");
|
||||
Assert.Equal(16, (await PlanAsync(card, destination)).FileCount);
|
||||
Assert.Equal(8, (await PlanAsync(card, destination, new ImportOptions { IncludeSubfolders = false })).FileCount);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task SessionsAreSplitPerCameraBodyAndNumberedFromOne()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
var dcim = Path.Combine(card, "DCIM", "100MEDIA");
|
||||
TestPhotos.WriteSeries(dcim, Start, 10, Interval, prefix: "CAN_");
|
||||
TestPhotos.WriteSeries(dcim, Start.AddHours(2), 10, Interval, firstSeed: 20, prefix: "CAN_", firstNumber: 11);
|
||||
TestPhotos.WriteSeries(dcim, Start.AddSeconds(5), 9, Interval, firstSeed: 40, prefix: "NIK_", make: "NIKON CORPORATION", model: "NIKON Z 8");
|
||||
|
||||
var plan = await PlanAsync(card, context.CreateTempDirectory("archive"));
|
||||
Assert.SequenceEqual(
|
||||
["2026/2026-09-28 Canon EOS R5", "2026/2026-09-28 Nikon Z 8", "2026/2026-09-28 Canon EOS R5 (2)"],
|
||||
plan.Sessions.Select(s => s.Folder));
|
||||
Assert.SequenceEqual([1, 2, 3], plan.Sessions.Select(s => s.Number));
|
||||
Assert.Equal("2026-09-28_20-00-00_0001.JPG", plan.Sessions[2].Items[0].FileName);
|
||||
Assert.Equal(3, plan.Sessions.Select(s => s.SessionKey).Distinct().Count());
|
||||
Assert.True(plan.Sessions.All(s => s.Items.All(i => i.SourcePath.Contains(s.CameraModel == "NIKON Z 8" ? "NIK_" : "CAN_"))));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task NamesTakenTwiceGetASuffix()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
TestPhotos.WriteSeries(card, Start, 3, Interval);
|
||||
var plan = await PlanAsync(card, context.CreateTempDirectory("archive"), new ImportOptions { FileTemplate = "{date}", MinimumSessionFrames = 1 });
|
||||
Assert.SequenceEqual(["2026-09-28.JPG", "2026-09-28_2.JPG", "2026-09-28_3.JPG"], plan.Sessions[0].Items.Select(i => i.FileName));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ImportedAndArchivedPhotosAreSkippedAfterAHashMatch()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
var files = TestPhotos.WriteSeries(card, Start, 10, Interval);
|
||||
var lookup = new FakeLookup();
|
||||
lookup.Imported.Add("IMG_0001.JPG");
|
||||
var archived = await ContentHash.Sha256Async(files[1]);
|
||||
lookup.Archive.Add((new FileInfo(files[1]).Length, Start + Interval, new ArchiveMatch(Guid.NewGuid(), archived)));
|
||||
lookup.Archive.Add((new FileInfo(files[2]).Length, Start + 2 * Interval, new ArchiveMatch(Guid.NewGuid(), new string('0', 64))));
|
||||
|
||||
var plan = await PlanAsync(card, context.CreateTempDirectory("archive"), lookup: lookup);
|
||||
var items = plan.Sessions[0].Items;
|
||||
Assert.Equal(ImportAction.SkipImported, items[0].Action);
|
||||
Assert.Equal(ImportAction.SkipInArchive, items[1].Action);
|
||||
Assert.Equal(lookup.Archive[0].Match.FrameId, items[1].ExistingFrameId);
|
||||
Assert.Equal(ImportAction.Copy, items[2].Action);
|
||||
Assert.Equal(await ContentHash.Sha256Async(files[2]), items[2].Sha256);
|
||||
Assert.Null(items[3].Sha256);
|
||||
Assert.Equal(8, plan.CopyCount);
|
||||
Assert.Equal("2026-09-28_18-00-10_0002.JPG", items[1].FileName);
|
||||
|
||||
var again = await PlanAsync(card, context.CreateTempDirectory("archive"), new ImportOptions { Duplicates = DuplicatePolicy.ImportAgain }, lookup);
|
||||
Assert.Equal(ImportAction.Copy, again.Sessions[0].Items[1].Action);
|
||||
Assert.Equal(lookup.Archive[0].Match.FrameId, again.Sessions[0].Items[1].ExistingFrameId);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ReplanningGivesStableKeysThatTitlesAndEarlierFoldersAttachTo()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
TestPhotos.WriteSeries(card, Start, 10, Interval);
|
||||
var destination = context.CreateTempDirectory("archive");
|
||||
var first = await PlanAsync(card, destination);
|
||||
var key = first.Sessions[0].SessionKey;
|
||||
Assert.Equal(key, (await PlanAsync(card, destination)).Sessions[0].SessionKey);
|
||||
|
||||
var titled = await PlanAsync(card, destination, new ImportOptions
|
||||
{
|
||||
FolderTemplate = "{year}/{date} {title}",
|
||||
Titles = new Dictionary<string, string> { [key] = "Harbour sunset" },
|
||||
});
|
||||
Assert.Equal("2026/2026-09-28 Harbour sunset", titled.Sessions[0].Folder);
|
||||
Assert.Equal("Harbour sunset", titled.Sessions[0].Title);
|
||||
|
||||
var lookup = new FakeLookup();
|
||||
lookup.SessionFolders[key] = "2026/earlier import";
|
||||
Assert.Equal("2026/earlier import", (await PlanAsync(card, destination, lookup: lookup)).Sessions[0].Folder);
|
||||
|
||||
var taken = new FakeLookup();
|
||||
taken.TakenFolders.Add("2026/2026-09-28 Canon EOS R5");
|
||||
Assert.Equal("2026/2026-09-28 Canon EOS R5 (2)", (await PlanAsync(card, destination, lookup: taken)).Sessions[0].Folder);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task UserMergeAndSplitReshapeSessions()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
TestPhotos.WriteSeries(card, Start, 10, Interval);
|
||||
TestPhotos.WriteSeries(card, Start.AddHours(3), 10, Interval, firstSeed: 30, firstNumber: 11);
|
||||
var destination = context.CreateTempDirectory("archive");
|
||||
Assert.Equal(2, (await PlanAsync(card, destination)).Sessions.Count);
|
||||
var merged = await PlanAsync(card, destination, new ImportOptions { MergeAt = new HashSet<string> { "IMG_0011.JPG" } });
|
||||
Assert.Equal(20, merged.Sessions.Single().Items.Count);
|
||||
Assert.Equal("2026-09-28_21-00-00_0011.JPG", merged.Sessions[0].Items[10].FileName);
|
||||
var split = await PlanAsync(card, destination, new ImportOptions { SplitBefore = new HashSet<string> { "IMG_0006.JPG" } });
|
||||
Assert.SequenceEqual([5, 5, 10], split.Sessions.Select(s => s.Items.Count));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task APhotoWithoutExifIsPlannedAtItsFileTime()
|
||||
{
|
||||
var card = context.CreateTempDirectory("card");
|
||||
TestPhotos.WriteSeries(card, Start, 8, Interval);
|
||||
var bare = Path.Combine(card, "BARE.JPG");
|
||||
File.WriteAllBytes(bare, [0xFF, 0xD8, 0xFF, 0xD9]);
|
||||
File.SetLastWriteTime(bare, Start.AddSeconds(85));
|
||||
var plan = await PlanAsync(card, context.CreateTempDirectory("archive"));
|
||||
var item = plan.Items.Single(i => i.SourcePath == "BARE.JPG");
|
||||
Assert.Equal(Start.AddSeconds(85), item.CapturedAt);
|
||||
// No camera identity: it forms its own body group rather than joining the Canon session.
|
||||
Assert.SequenceEqual([8, 1], plan.Sessions.Select(s => s.Items.Count));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using Titano.Core.Import;
|
||||
|
||||
namespace Titano.Core.Tests.Import;
|
||||
|
||||
/// <summary>An <see cref="IImportLookup"/> answered from in-memory sets.</summary>
|
||||
internal sealed class FakeLookup : IImportLookup
|
||||
{
|
||||
public HashSet<string> Imported { get; } = [];
|
||||
public List<(long Size, DateTime Captured, ArchiveMatch Match)> Archive { get; } = [];
|
||||
public Dictionary<string, string> SessionFolders { get; } = [];
|
||||
public HashSet<string> TakenFolders { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public int CandidateQueries { get; private set; }
|
||||
|
||||
public Task<bool> WasImportedAsync(CardFile file, CancellationToken cancellationToken) => Task.FromResult(Imported.Contains(file.RelativePath));
|
||||
|
||||
public Task<IReadOnlyList<ArchiveMatch>> FindCandidatesAsync(long size, DateTime capturedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
CandidateQueries++;
|
||||
return Task.FromResult<IReadOnlyList<ArchiveMatch>>(Archive.Where(a => a.Size == size && a.Captured == capturedAt).Select(a => a.Match).ToList());
|
||||
}
|
||||
|
||||
public Task<string?> FindSessionFolderAsync(string sessionKey, CancellationToken cancellationToken) =>
|
||||
Task.FromResult(SessionFolders.GetValueOrDefault(sessionKey));
|
||||
|
||||
public bool IsFolderTaken(string folder) => TakenFolders.Contains(folder);
|
||||
}
|
||||
|
||||
internal static class ImportTestKit
|
||||
{
|
||||
public static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
|
||||
|
||||
/// <summary>Plans an import of <paramref name="card"/> into <paramref name="destination"/>.</summary>
|
||||
public static async Task<ImportPlan> PlanAsync(string card, string destination, ImportOptions? options = null, FakeLookup? lookup = null)
|
||||
{
|
||||
options ??= new ImportOptions();
|
||||
var problems = new List<ImportProblem>();
|
||||
var photos = CardScanner.ReadMetadata(card, CardScanner.Find(card, options.IncludeSubfolders), problems);
|
||||
return await ImportPlanner.PlanAsync(card, destination, photos, problems, options, lookup ?? new FakeLookup());
|
||||
}
|
||||
|
||||
/// <summary>Every file below <paramref name="root"/> (relative, forward slashes), excluding Titano's metadata folders.</summary>
|
||||
public static List<string> Files(string root) =>
|
||||
Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)
|
||||
.Select(f => Path.GetRelativePath(root, f).Replace('\\', '/'))
|
||||
.Where(f => !f.StartsWith(".titano/", StringComparison.Ordinal))
|
||||
.Order(StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using Titano.Core.Import;
|
||||
using Titano.Testing;
|
||||
|
||||
namespace Titano.Core.Tests.Import;
|
||||
|
||||
[Category("import")]
|
||||
public sealed class SessionSplitterTests
|
||||
{
|
||||
private static readonly DateTime Start = new(2026, 9, 28, 18, 0, 0);
|
||||
private static readonly TimeSpan Gap = TimeSpan.FromMinutes(45);
|
||||
|
||||
/// <summary>Capture times from a compact description: runs of frames 10 s apart, separated by pauses in minutes.</summary>
|
||||
private static List<DateTime> Times(params (int Frames, int PauseBeforeMinutes)[] runs)
|
||||
{
|
||||
var times = new List<DateTime>();
|
||||
var t = Start;
|
||||
foreach (var (frames, pause) in runs)
|
||||
{
|
||||
t = t.AddMinutes(pause);
|
||||
for (var i = 0; i < frames; i++)
|
||||
{
|
||||
times.Add(t);
|
||||
t = t.AddSeconds(10);
|
||||
}
|
||||
}
|
||||
return times;
|
||||
}
|
||||
|
||||
private static string Describe(IReadOnlyList<SessionRange> ranges) => string.Join(",", ranges.Select(r => r.Count));
|
||||
|
||||
[Theory]
|
||||
[Case(new[] { 30 }, new[] { 0 }, "30", Name = "one session")]
|
||||
[Case(new[] { 30, 40 }, new[] { 0, 60 }, "30,40", Name = "pause longer than the gap splits")]
|
||||
[Case(new[] { 30, 40 }, new[] { 0, 44 }, "70", Name = "pause shorter than the gap does not")]
|
||||
[Case(new[] { 3, 30 }, new[] { 0, 90 }, "33", Name = "leading test shots join the next session")]
|
||||
[Case(new[] { 30, 3 }, new[] { 0, 90 }, "33", Name = "trailing test shots join the previous session")]
|
||||
[Case(new[] { 30, 2, 30 }, new[] { 0, 90, 90 }, "32,30", Name = "small middle group joins the previous session")]
|
||||
[Case(new[] { 2, 3 }, new[] { 0, 90 }, "5", Name = "only small groups make one session")]
|
||||
public void AutomaticSessions(int[] frames, int[] pauses, string expected)
|
||||
{
|
||||
var times = Times(frames.Zip(pauses).ToArray());
|
||||
Assert.Equal(expected, Describe(SessionSplitter.Split(times, Gap, 8)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APauseOfExactlyTheGapDoesNotSplit()
|
||||
{
|
||||
var times = new List<DateTime> { Start };
|
||||
for (var i = 1; i < 10; i++) times.Add(Start.AddSeconds(i));
|
||||
for (var i = 0; i < 10; i++) times.Add(times[9] + Gap + TimeSpan.FromSeconds(i));
|
||||
Assert.Equal("20", Describe(SessionSplitter.Split(times, Gap, 8)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void UserSplitsAndMergesOverrideTheRules()
|
||||
{
|
||||
var times = Times((30, 0), (40, 60));
|
||||
// Split before frame 25 (a 5-frame group the user wants on its own) and merge the automatic break at 30.
|
||||
var ranges = SessionSplitter.Split(times, Gap, 8, i => i == 25 ? true : i == 30 ? false : null);
|
||||
Assert.Equal("25,45", Describe(ranges));
|
||||
Assert.Equal(25, ranges[1].Start);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AForcedSmallSessionIsNotMergedAway()
|
||||
{
|
||||
var times = Times((30, 0));
|
||||
Assert.Equal("28,2", Describe(SessionSplitter.Split(times, Gap, 8, i => i == 28 ? true : null)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NoFramesNoSessions() => Assert.Empty(SessionSplitter.Split([], Gap, 8));
|
||||
}
|
||||
Reference in new issue
Block a user