fix(archive): weigh replaced originals against the archive, not this index
Owner review 2, phase 4b, fix round 2 (trash and sync). - Emptying a Replaced entry decided "conversion deleted" from this host's index alone, so a conversion another host made, or one this index had not read yet, could be taken for deleted and its originals erased without re-hashing it. The survey now reads the archive: the sequence's sidecar where the index or the entry says it is, in the trash, or found by walking the archive's folders. Its Replaced tombstones are followed and every conversion it holds is re-hashed. A conversion counts as deleted only when the sidecar shows it (a Deleted tombstone, or the frame gone while every frame is hashed and no photo of the folder is unlisted), when the sequence is in the trash, or when no sidecar of it exists and this index has no row either. Whatever the archive cannot tell, or a converted frame whose file changed since, keeps the entry, and the job log says why (changed after the plan, sequence busy, conversion no longer matches, conversion not located). - The erase weighs a Replaced entry under the entry's and the sequence's leases and settles the sequence first (an undo waiting for that entry's lease completes), so the entry it reads again and fingerprints is the one it erases. Read before the settle, a consumed entry's folder vanished under the erase. - Each sidecar is indexed once per survey: one conversion's entry (every frame it swapped) is weighed in one pass, not one pass over the sidecar per photo. - The desktop Empty confirmation names every entry whose conversion was deleted first, with why, wherever the trash lists it; the detail scrolls when long. The panel lists them first as well. - Download originals queues the passes a frame swapped to another encoding cleared (light analysis, the deep analysis it had, thumbnails, proxies), as any replacement does. - The job log lines of uploads and downloads are resources (en, it). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> (cherry picked from commit 27d9a512183e64da9a3d7fa69a267470ca3ce744)
This commit is contained in:
1 parent
c669d0504d
commit
48a0ce3e02
15 files changed
+834
-164
No files matched your search
+13
-3
@@ -177,8 +177,8 @@
|
||||
<data name="Archive.Error.SourceGone" xml:space="preserve"><value>The card or folder {0} is no longer available.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.StalePlan" xml:space="preserve"><value>The archive changed after this plan was made.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.TrashChanged" xml:space="preserve"><value>The trash changed after it was checked; nothing was erased.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Error.TrashKept.one" xml:space="preserve"><value>{0:N0} trash entry was kept: it changed, or a replacement no longer matches.</value><comment>long: error sentence of an emptying job</comment></data>
|
||||
<data name="Archive.Error.TrashKept.other" xml:space="preserve"><value>{0:N0} trash entries were kept: they changed, or a replacement no longer matches.</value><comment>long: error sentence of an emptying job</comment></data>
|
||||
<data name="Archive.Error.TrashKept.one" xml:space="preserve"><value>{0:N0} trash entry was kept; the job log says why.</value><comment>long: error sentence of an emptying job</comment></data>
|
||||
<data name="Archive.Error.TrashKept.other" xml:space="preserve"><value>{0:N0} trash entries were kept; the job log says why.</value><comment>long: error sentence of an emptying job</comment></data>
|
||||
<data name="Archive.Error.TrashMismatch" xml:space="preserve"><value>No file of this trash entry matches its record any more; it stays in the trash.</value><comment>long: error sentence</comment></data>
|
||||
<data name="Archive.Failed.Access" xml:space="preserve"><value>Chronos was denied access to the archive.</value><comment>long</comment></data>
|
||||
<data name="Archive.Failed.Files" xml:space="preserve"><value>A file of the archive could not be read or written.</value><comment>long</comment></data>
|
||||
@@ -259,9 +259,19 @@
|
||||
<data name="Archive.Log.SequenceMissing" xml:space="preserve"><value>Sequence not found: {0}</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.StepFailed" xml:space="preserve"><value>Clean-up step {0} failed: {1}</value><comment>long: job log line; {0} step name (job-history, audit…); {1} error message</comment></data>
|
||||
<data name="Archive.Log.StepUnknown" xml:space="preserve"><value>Clean-up step unknown here: {0}</value><comment>long: job log line; {0} step name</comment></data>
|
||||
<data name="Archive.Log.SyncChanged" xml:space="preserve"><value>{0}: changed since it was archived, not uploaded</value><comment>long: job log line of an upload; {0} file name</comment></data>
|
||||
<data name="Archive.Log.SyncDeletedHere" xml:space="preserve"><value>{0}: deleted on this PC, not downloaded</value><comment>long: job log line of a download; {0} file name</comment></data>
|
||||
<data name="Archive.Log.SyncDeletedThere" xml:space="preserve"><value>{0}: deleted on the server, not uploaded</value><comment>long: job log line of an upload; {0} file name</comment></data>
|
||||
<data name="Archive.Log.SyncFrameChanged" xml:space="preserve"><value>{0}: its frame here changed meanwhile, not downloaded</value><comment>long: job log line of a download; {0} file name of the server's copy of a photo this PC holds in another encoding</comment></data>
|
||||
<data name="Archive.Log.SyncMissing" xml:space="preserve"><value>{0}: missing, not uploaded</value><comment>long: job log line of an upload; {0} file name</comment></data>
|
||||
<data name="Archive.Log.SyncNoPreview.one" xml:space="preserve"><value>{0:N0} frame has neither proxy nor thumbnail on the server</value><comment>long: job log line of a proxy download</comment></data>
|
||||
<data name="Archive.Log.SyncNoPreview.other" xml:space="preserve"><value>{0:N0} frames have neither proxy nor thumbnail on the server</value><comment>long: job log line of a proxy download</comment></data>
|
||||
<data name="Archive.Log.TrashEmptied" xml:space="preserve"><value>Entries erased: {0}, size: {1}</value><comment>long: job log line; {0} count; {1} size</comment></data>
|
||||
<data name="Archive.Log.TrashEntry" xml:space="preserve"><value>In the trash since {0}: {1} ({2})</value><comment>long: job log line; {0} date; {1} sequence title; {2} size</comment></data>
|
||||
<data name="Archive.Log.TrashKept" xml:space="preserve"><value>Kept in the trash: {0}</value><comment>long: job log line; {0} sequence title</comment></data>
|
||||
<data name="Archive.Log.TrashKept" xml:space="preserve"><value>Kept in the trash, changed after the plan: {0}</value><comment>long: job log line of an emptying; {0} sequence title</comment></data>
|
||||
<data name="Archive.Log.TrashKeptBusy" xml:space="preserve"><value>Kept in the trash, its sequence was busy: {0}</value><comment>long: job log line of an emptying; {0} sequence title</comment></data>
|
||||
<data name="Archive.Log.TrashKeptMismatch" xml:space="preserve"><value>Kept in the trash, a conversion of its originals no longer matches: {0}</value><comment>long: job log line of an emptying; {0} sequence title</comment></data>
|
||||
<data name="Archive.Log.TrashKeptUnknown" xml:space="preserve"><value>Kept in the trash, the conversion of its originals could not be located; scan the archive: {0}</value><comment>long: job log line of an emptying; {0} sequence title</comment></data>
|
||||
<data name="Archive.Log.TrashOffline" xml:space="preserve"><value>Trash not checked: the archive cannot be reached</value><comment>long: job log line</comment></data>
|
||||
<data name="Archive.Log.TrashReport" xml:space="preserve"><value>Old trash entries: {0}, size: {1}; nothing was deleted</value><comment>long: job log line; entries older than the clean-up's age limit</comment></data>
|
||||
<data name="Archive.Log.VerifyDone" xml:space="preserve"><value>Photos checked: {0}, intact: {1}, hashed: {2}, with problems: {3}</value><comment>long: job log line</comment></data>
|
||||
|
||||
+13
-3
@@ -171,8 +171,8 @@
|
||||
<data name="Archive.Error.SourceGone" xml:space="preserve"><value>La scheda o cartella {0} non è più disponibile.</value></data>
|
||||
<data name="Archive.Error.StalePlan" xml:space="preserve"><value>L'archivio è cambiato dopo la creazione del piano.</value></data>
|
||||
<data name="Archive.Error.TrashChanged" xml:space="preserve"><value>Il cestino è cambiato dopo il controllo; non è stato cancellato nulla.</value></data>
|
||||
<data name="Archive.Error.TrashKept.one" xml:space="preserve"><value>{0:N0} elemento del cestino è stato mantenuto: è cambiato, o una sostituzione non corrisponde più.</value></data>
|
||||
<data name="Archive.Error.TrashKept.other" xml:space="preserve"><value>{0:N0} elementi del cestino sono stati mantenuti: sono cambiati, o una sostituzione non corrisponde più.</value></data>
|
||||
<data name="Archive.Error.TrashKept.one" xml:space="preserve"><value>{0:N0} elemento del cestino è stato mantenuto; il registro del job dice perché.</value></data>
|
||||
<data name="Archive.Error.TrashKept.other" xml:space="preserve"><value>{0:N0} elementi del cestino sono stati mantenuti; il registro del job dice perché.</value></data>
|
||||
<data name="Archive.Error.TrashMismatch" xml:space="preserve"><value>Nessun file di questo elemento del cestino corrisponde più al suo registro; resta nel cestino.</value></data>
|
||||
<data name="Archive.Failed.Access" xml:space="preserve"><value>Chronos non ha accesso all'archivio.</value></data>
|
||||
<data name="Archive.Failed.Files" xml:space="preserve"><value>Non è stato possibile leggere o scrivere un file dell'archivio.</value></data>
|
||||
@@ -253,9 +253,19 @@
|
||||
<data name="Archive.Log.SequenceMissing" xml:space="preserve"><value>Sequenza non trovata: {0}</value></data>
|
||||
<data name="Archive.Log.StepFailed" xml:space="preserve"><value>Passo di pulizia {0} non riuscito: {1}</value></data>
|
||||
<data name="Archive.Log.StepUnknown" xml:space="preserve"><value>Passo di pulizia sconosciuto qui: {0}</value></data>
|
||||
<data name="Archive.Log.SyncChanged" xml:space="preserve"><value>{0}: cambiato da quando è stato archiviato, non caricato</value></data>
|
||||
<data name="Archive.Log.SyncDeletedHere" xml:space="preserve"><value>{0}: eliminato su questo PC, non scaricato</value></data>
|
||||
<data name="Archive.Log.SyncDeletedThere" xml:space="preserve"><value>{0}: eliminato sul server, non caricato</value></data>
|
||||
<data name="Archive.Log.SyncFrameChanged" xml:space="preserve"><value>{0}: il suo fotogramma qui è cambiato nel frattempo, non scaricato</value></data>
|
||||
<data name="Archive.Log.SyncMissing" xml:space="preserve"><value>{0}: mancante, non caricato</value></data>
|
||||
<data name="Archive.Log.SyncNoPreview.one" xml:space="preserve"><value>{0:N0} fotogramma non ha né proxy né miniatura sul server</value></data>
|
||||
<data name="Archive.Log.SyncNoPreview.other" xml:space="preserve"><value>{0:N0} fotogrammi non hanno né proxy né miniatura sul server</value></data>
|
||||
<data name="Archive.Log.TrashEmptied" xml:space="preserve"><value>Elementi cancellati: {0}, dimensione: {1}</value></data>
|
||||
<data name="Archive.Log.TrashEntry" xml:space="preserve"><value>Nel cestino dal {0}: {1} ({2})</value></data>
|
||||
<data name="Archive.Log.TrashKept" xml:space="preserve"><value>Mantenuto nel cestino: {0}</value></data>
|
||||
<data name="Archive.Log.TrashKept" xml:space="preserve"><value>Mantenuto nel cestino, cambiato dopo il piano: {0}</value></data>
|
||||
<data name="Archive.Log.TrashKeptBusy" xml:space="preserve"><value>Mantenuto nel cestino, la sua sequenza era occupata: {0}</value></data>
|
||||
<data name="Archive.Log.TrashKeptMismatch" xml:space="preserve"><value>Mantenuto nel cestino, una conversione dei suoi originali non corrisponde più: {0}</value></data>
|
||||
<data name="Archive.Log.TrashKeptUnknown" xml:space="preserve"><value>Mantenuto nel cestino, la conversione dei suoi originali non è stata trovata; esegui una scansione dell'archivio: {0}</value></data>
|
||||
<data name="Archive.Log.TrashOffline" xml:space="preserve"><value>Cestino non controllato: l'archivio non è raggiungibile</value></data>
|
||||
<data name="Archive.Log.TrashReport" xml:space="preserve"><value>Voci vecchie nel cestino: {0}, dimensione: {1}; nulla è stato eliminato</value></data>
|
||||
<data name="Archive.Log.VerifyDone" xml:space="preserve"><value>Foto controllate: {0}, integre: {1}, con impronta nuova: {2}, con problemi: {3}</value></data>
|
||||
|
||||
@@ -41,17 +41,21 @@ public static class TrashEmptyPlans
|
||||
/// <summary>
|
||||
/// The confirmation of erasing the planned entries for good; null when the plan holds none. An entry of replaced originals
|
||||
/// whose conversion was deleted since says so (<see cref="TrashEntry.ConversionDeleted"/>): erasing it may erase the last
|
||||
/// copy of those photos.
|
||||
/// copy of those photos, so every such entry is named first, with why, wherever the trash lists it; the other entries
|
||||
/// fill the lines left (<see cref="DeletePlans.ListedLines"/>) and the rest are counted.
|
||||
/// </summary>
|
||||
public static ConfirmRequest? Request(TrashEmptyPlan plan)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(plan);
|
||||
if (plan.Entries.Count == 0) return null;
|
||||
var count = plan.Entries.Count.ToString(CultureInfo.InvariantCulture);
|
||||
var lines = plan.Entries.Take(DeletePlans.ListedLines).Select(e => Loc.L("Archive.Delete.Line", e.Title,
|
||||
var lastCopies = plan.Entries.Where(e => e.ConversionDeleted).ToList();
|
||||
var others = plan.Entries.Where(e => !e.ConversionDeleted).ToList();
|
||||
var listed = Math.Max(0, DeletePlans.ListedLines - lastCopies.Count);
|
||||
var lines = lastCopies.Concat(others.Take(listed)).Select(e => Loc.L("Archive.Delete.Line", e.Title,
|
||||
KindText(e.Kind) + Loc.Separator + Loc.Plural("Shell.Status.Photos", e.Frames)
|
||||
+ (e.ConversionDeleted ? Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted") : string.Empty), Formatting.Bytes(e.Bytes))).ToList();
|
||||
if (plan.Entries.Count > DeletePlans.ListedLines) lines.Add(Loc.Plural("Archive.Delete.More", plan.Entries.Count - DeletePlans.ListedLines));
|
||||
if (others.Count > listed) lines.Add(Loc.Plural("Archive.Delete.More", others.Count - listed));
|
||||
return new ConfirmRequest
|
||||
{
|
||||
Heading = Loc.Plural("Archive.Trash.EmptyHeading", plan.Entries.Count),
|
||||
|
||||
@@ -12,10 +12,14 @@
|
||||
<StackPanel Margin="24,18,24,20">
|
||||
<TextBlock Text="{Binding Heading}" Style="{StaticResource Text.Title}" TextWrapping="Wrap" TextTrimming="None" />
|
||||
<TextBlock Text="{Binding Message}" Margin="0,8,0,0" Foreground="{DynamicResource Brush.TextMuted}" TextWrapping="Wrap" TextTrimming="None" LineHeight="18" />
|
||||
<!-- A detail longer than the card (every trash entry that may hold the last copy of its photos is named) scrolls, so the
|
||||
buttons stay on a 768 px screen. -->
|
||||
<Border Style="{StaticResource Border.Card}" Margin="0,16,0,0" Padding="12,10" ToolTip="{Binding DetailToolTip}"
|
||||
Visibility="{Binding Detail, Converter={x:Static ui:NullToCollapsedConverter.Instance}}">
|
||||
<TextBlock x:Name="DetailText" Text="{Binding Detail}" FontFamily="{StaticResource Font.Mono}" FontSize="{StaticResource Size.Secondary}" TextWrapping="Wrap" TextTrimming="None" LineHeight="18"
|
||||
FlowDirection="LeftToRight" />
|
||||
<ScrollViewer MaxHeight="288" VerticalScrollBarVisibility="Auto" HorizontalScrollBarVisibility="Disabled" Focusable="False">
|
||||
<TextBlock x:Name="DetailText" Text="{Binding Detail}" FontFamily="{StaticResource Font.Mono}" FontSize="{StaticResource Size.Secondary}" TextWrapping="Wrap" TextTrimming="None" LineHeight="18"
|
||||
FlowDirection="LeftToRight" />
|
||||
</ScrollViewer>
|
||||
</Border>
|
||||
<StackPanel x:Name="Typing" Margin="0,16,0,0" Visibility="Collapsed">
|
||||
<TextBlock Text="{Binding TypedPrompt}" Foreground="{DynamicResource Brush.TextMuted}" TextWrapping="Wrap" />
|
||||
|
||||
@@ -627,12 +627,13 @@ public sealed partial class FrameTrash
|
||||
/// <paramref name="sha256"/> and has no file, <paramref name="staged"/> takes its place (<see cref="ReplaceAsync"/>,
|
||||
/// <see cref="FrameSwap.Missing"/>): the same id and index, the user's flags, a record read from the file as filing reads
|
||||
/// one (named as the frame with the file's extension), the old content as its source and a tombstone naming the new one,
|
||||
/// so the photo is neither filed twice nor missing. Nothing goes to the trash. False, nothing touched, when the frame
|
||||
/// changed or its file is back.
|
||||
/// so the photo is neither filed twice nor missing. Nothing goes to the trash. Returns the sequence as it is now and its
|
||||
/// statuses before, whose passes (<see cref="FollowUps"/>) the swap cleared as any replacement does; null, nothing
|
||||
/// touched, when the frame changed or its file is back.
|
||||
/// </summary>
|
||||
/// <param name="staged">The verified copy, inside the archive (its staging); moved in.</param>
|
||||
/// <param name="fileName">The copy's name on the other side, whose extension the frame's file takes.</param>
|
||||
internal async Task<bool> ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256,
|
||||
internal async Task<ReplaceResult?> ReplaceMissingAsync(Archive archive, Guid sequenceId, Guid frameId, string sha256, string staged, string stagedSha256,
|
||||
string fileName, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(archive);
|
||||
@@ -641,10 +642,9 @@ public sealed partial class FrameTrash
|
||||
var metadata = Core.Metadata.MetadataReader.Read(staged, cancellationToken);
|
||||
var extension = Path.GetExtension(fileName);
|
||||
var swap = new FrameSwap(frameId, paths.ToRelative(staged), [], Named, Read) { Missing = true };
|
||||
var replaced = await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null,
|
||||
return await ReplaceAsync(archive, sequenceId, null, [swap], null, null, null,
|
||||
(manifest, _, _) => Task.FromResult(manifest.Frames.Any(f => f.Id == frameId && string.Equals(f.Sha256, sha256, StringComparison.OrdinalIgnoreCase))),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
return replaced is not null;
|
||||
|
||||
string Named(SidecarFrame current, SequenceManifest _) => extension.Length > 1 ? Path.ChangeExtension(current.File, extension) : current.File;
|
||||
|
||||
|
||||
@@ -10,8 +10,9 @@ namespace Chronos.Data.Archiving.Jobs;
|
||||
/// <see cref="JobKind.EmptyTrash"/> (owner review 2, R6): erases exactly the entries of a confirmed plan
|
||||
/// (<see cref="EmptyTrashParameters"/>) and fails, erasing nothing, when the trash no longer matches the plan's
|
||||
/// fingerprint (<see cref="StaleTrashPlanException"/>). Checkpoint: <c>done:<n></c>, the entries done; a resumed job
|
||||
/// checks each remaining entry against its own fingerprint. Entries kept (a replacement that no longer matches, an entry
|
||||
/// changed since) are logged and fail the job at the end, after the others are erased.
|
||||
/// checks each remaining entry against its own fingerprint. Entries kept (an entry changed since, a conversion of replaced
|
||||
/// originals that no longer matches, cannot be located or is busy) are logged each with why, and fail the job at the end,
|
||||
/// after the others are erased.
|
||||
/// </summary>
|
||||
internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash trash) : IJobHandler
|
||||
{
|
||||
@@ -31,7 +32,7 @@ internal sealed class EmptyTrashJobHandler(ChronosDatabase db, SequenceTrash tra
|
||||
var result = await trash.EmptyAsync(archive, parameters, start,
|
||||
(n, total) => context.CheckpointAsync(Prefix + n.ToString(CultureInfo.InvariantCulture), n, total), context.Cancellation).ConfigureAwait(false);
|
||||
context.Log(Loc.L("Archive.Log.TrashEmptied", result.Erased, CleanupSteps.Megabytes(result.Bytes)));
|
||||
foreach (var title in result.Refused) context.Log(Loc.L("Archive.Log.TrashKept", title));
|
||||
foreach (var kept in result.Refused) context.Log(kept.LogLine);
|
||||
if (result.Refused.Count > 0) throw new InvalidOperationException(Loc.Plural("Archive.Error.TrashKept", result.Refused.Count));
|
||||
}
|
||||
}
|
||||
@@ -21,8 +21,43 @@ public sealed record TrashEmptyPlan(Guid ArchiveId, IReadOnlyList<TrashEntry> En
|
||||
public int Photos => Entries.Sum(e => e.Frames);
|
||||
}
|
||||
|
||||
/// <summary>What an emptying erased, and the entries it kept (a replacement that no longer matches, an entry changed since the plan).</summary>
|
||||
public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList<string> Refused);
|
||||
/// <summary>What an emptying erased, and the entries it kept, each with why (<see cref="TrashKept"/>).</summary>
|
||||
public sealed record TrashEmptyResult(int Erased, long Bytes, IReadOnlyList<TrashKept> Refused);
|
||||
|
||||
/// <summary>An entry an emptying kept in the trash: its title and why.</summary>
|
||||
public sealed record TrashKept(string Title, TrashKeptReason Reason)
|
||||
{
|
||||
/// <summary>"Kept in the trash, changed after the plan: Etna", a job log line in the current culture.</summary>
|
||||
public string LogLine => Loc.L(Reason switch
|
||||
{
|
||||
TrashKeptReason.Mismatch => "Archive.Log.TrashKeptMismatch",
|
||||
TrashKeptReason.Unknown => "Archive.Log.TrashKeptUnknown",
|
||||
TrashKeptReason.Busy => "Archive.Log.TrashKeptBusy",
|
||||
_ => "Archive.Log.TrashKept",
|
||||
}, Title);
|
||||
}
|
||||
|
||||
/// <summary>Why an emptying kept an entry in the trash.</summary>
|
||||
public enum TrashKeptReason
|
||||
{
|
||||
/// <summary>The entry changed since the emptying was planned (restored, or deleted again, meanwhile), or is gone.</summary>
|
||||
Changed,
|
||||
|
||||
/// <summary>
|
||||
/// A conversion of its originals (owner review 2, R7) no longer matches its content, in its sequence or in the trash, or
|
||||
/// its frame holds other content now (its file changed since the conversion).
|
||||
/// </summary>
|
||||
Mismatch,
|
||||
|
||||
/// <summary>
|
||||
/// The archive cannot tell where a conversion of its originals is (a sidecar that cannot be read, a frame never hashed, a
|
||||
/// sequence the index has and the archive does not): a scan of the archive settles it.
|
||||
/// </summary>
|
||||
Unknown,
|
||||
|
||||
/// <summary>Its sequence is busy: another writer holds it, or a change to it is not settled yet.</summary>
|
||||
Busy,
|
||||
}
|
||||
|
||||
/// <summary>The trash changed after the emptying was planned (an entry came, went or changed); nothing was erased.</summary>
|
||||
public sealed class StaleTrashPlanException() : InvalidOperationException(Loc.L("Archive.Error.TrashChanged"));
|
||||
@@ -53,7 +88,9 @@ public sealed partial class SequenceTrash
|
||||
var (files, bytes) = Directory.Exists(folder) ? Measure(folder, cancellationToken) : (0, 0L);
|
||||
return e with { Files = files, Bytes = bytes };
|
||||
}).ToList(), cancellationToken).ConfigureAwait(false);
|
||||
for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(paths, measured[i], cancellationToken).ConfigureAwait(false);
|
||||
// The sequences of replaced entries, found once for the whole plan.
|
||||
var located = new Dictionary<Guid, Located>();
|
||||
for (var i = 0; i < measured.Count; i++) measured[i] = await PlannedAsync(archive, paths, measured[i], located, cancellationToken).ConfigureAwait(false);
|
||||
var unknown = names?.Where(n => !listed.Any(e => e.Name == n)).Distinct(StringComparer.Ordinal).ToList() ?? [];
|
||||
return new TrashEmptyPlan(archive.Id, measured, measured.Sum(e => e.Bytes), Fingerprint(measured), unknown);
|
||||
}
|
||||
@@ -87,9 +124,11 @@ public sealed partial class SequenceTrash
|
||||
/// hold exactly the planned entries (<see cref="EmptyTrashParameters.Fingerprint"/>), else nothing is erased; a run
|
||||
/// resumed after a stop checks each remaining entry against its own fingerprint, keeps one that changed and skips one
|
||||
/// that is gone, and each entry is checked against its fingerprint once more under its lease, just before it goes. A
|
||||
/// <see cref="TrashKind.Replaced"/> entry is erased only while every replacement still matches its SHA-256 where it is now
|
||||
/// (re-hashed now: in the sequence, or in the trash when its conversion was deleted, as the dry run said); otherwise it is
|
||||
/// kept. Leftovers of an interrupted erase are deleted first.
|
||||
/// <see cref="TrashKind.Replaced"/> entry is weighed under its sequence's lease as well, against the archive as it is then
|
||||
/// (<see cref="Survey"/>): it is erased only while every conversion of its originals still matches its SHA-256 where the
|
||||
/// archive shows it (re-hashed now: in the sequence, or a copy the trash holds when its conversion was deleted, as the
|
||||
/// dry run said); otherwise, or when the archive cannot tell, it is kept, with why. Leftovers of an interrupted erase are
|
||||
/// deleted first.
|
||||
/// </summary>
|
||||
/// <param name="progress">Entries done so far (the resume point) and their number.</param>
|
||||
/// <exception cref="StaleTrashPlanException">The trash no longer holds exactly the planned entries; nothing was erased.</exception>
|
||||
@@ -104,7 +143,7 @@ public sealed partial class SequenceTrash
|
||||
var items = parameters.Entries;
|
||||
var current = await PlanEmptyAsync(archive, [.. items.Skip(start).Select(i => i.Name)], cancellationToken).ConfigureAwait(false);
|
||||
if (start == 0 && (current.Fingerprint != parameters.Fingerprint || current.Unknown.Count > 0)) throw new StaleTrashPlanException();
|
||||
var refused = new List<string>();
|
||||
var refused = new List<TrashKept>();
|
||||
int erased = 0;
|
||||
long bytes = 0;
|
||||
for (var i = start; i < items.Count; i++)
|
||||
@@ -115,17 +154,17 @@ public sealed partial class SequenceTrash
|
||||
{
|
||||
if (EntryFingerprint(entry) != item.Fingerprint)
|
||||
{
|
||||
refused.Add(entry.Title);
|
||||
refused.Add(new TrashKept(entry.Title, TrashKeptReason.Changed));
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
|
||||
}
|
||||
else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false))
|
||||
else if (await EraseAsync(archive, paths, entry, item.Fingerprint, cancellationToken).ConfigureAwait(false) is { } reason)
|
||||
{
|
||||
erased++;
|
||||
bytes += entry.Bytes;
|
||||
refused.Add(new TrashKept(entry.Title, reason));
|
||||
}
|
||||
else
|
||||
{
|
||||
refused.Add(entry.Title);
|
||||
erased++;
|
||||
bytes += entry.Bytes;
|
||||
}
|
||||
}
|
||||
await progress(i + 1, items.Count).ConfigureAwait(false);
|
||||
@@ -136,49 +175,98 @@ public sealed partial class SequenceTrash
|
||||
/// <summary>
|
||||
/// Erases one entry under its lease, once its record read and its folder measured again under it still give the
|
||||
/// planned <paramref name="fingerprint"/> (another host sharing the archive may have restored the entry and deleted the
|
||||
/// same sequence again since); false (logged) when it changed, or a replaced entry's replacements no longer match.
|
||||
/// same sequence again since). A replaced entry is weighed under its sequence's lease too, so no writer of the sequence
|
||||
/// moves its conversions meanwhile, and only once what a writer left unsettled in the sequence is settled (an undo whose
|
||||
/// completion waited for this entry's lease takes photos out of it): the archive must tell where each conversion is (as
|
||||
/// the dry run said, which the fingerprint holds), and each must still match. Returns null once erased, else why the
|
||||
/// entry is kept (logged).
|
||||
/// </summary>
|
||||
private async Task<bool> EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken)
|
||||
private async Task<TrashKeptReason?> EraseAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string fingerprint, CancellationToken cancellationToken)
|
||||
{
|
||||
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, EntryLease(entry.Name), _writer.Options.Owner, _writer.Options.LeaseWait,
|
||||
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var (folder, record) = Location(paths, entry);
|
||||
if (!Directory.Exists(folder) || !File.Exists(record)) return false;
|
||||
if (await RereadAsync(paths, entry, cancellationToken).ConfigureAwait(false) is not { } now || EntryFingerprint(now) != fingerprint)
|
||||
if (!Directory.Exists(folder) || !File.Exists(record)) return TrashKeptReason.Changed;
|
||||
IAsyncDisposable? sequence = null;
|
||||
if (entry.Kind == TrashKind.Replaced)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
|
||||
return false;
|
||||
try
|
||||
{
|
||||
sequence = await _writer.LockAsync(paths, entry.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseUnavailableException ex)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept: its sequence is busy", ex);
|
||||
return TrashKeptReason.Busy;
|
||||
}
|
||||
}
|
||||
if (entry.Kind == TrashKind.Replaced && !await ReplacementsHoldAsync(archive, paths, entry, cancellationToken).ConfigureAwait(false))
|
||||
await using (sequence)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept: a replacement of its originals no longer matches");
|
||||
return false;
|
||||
// Settled before the entry is read again, so what is read is what an erase takes.
|
||||
var located = sequence is null ? null : await LocateAsync(archive, paths, entry, EntryLease(entry.Name), cancellationToken).ConfigureAwait(false);
|
||||
var (now, survey) = Reread(archive, paths, entry, located, cancellationToken);
|
||||
if (survey?.Kept is { } kept)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept ({kept}): the archive does not show the conversions of its originals whole");
|
||||
return kept;
|
||||
}
|
||||
if (now is null || EntryFingerprint(now) != fingerprint)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} changed since the emptying was planned; it is kept");
|
||||
return TrashKeptReason.Changed;
|
||||
}
|
||||
if (survey is not null && !await MatchAsync(survey, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {entry.Name} is kept: a conversion of its originals no longer matches");
|
||||
return TrashKeptReason.Mismatch;
|
||||
}
|
||||
var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
|
||||
Directory.Move(folder, aside);
|
||||
File.Delete(record);
|
||||
Erase(aside);
|
||||
Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased"
|
||||
+ (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty));
|
||||
return null;
|
||||
}
|
||||
var aside = Path.Combine(TrashDirectory(paths), ErasingFolder, Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
|
||||
Directory.Move(folder, aside);
|
||||
File.Delete(record);
|
||||
Erase(aside);
|
||||
Log.Info("archive", $"Trash entry {entry.Name} ({entry.Title}, {entry.Files} files, {entry.Bytes} bytes) erased"
|
||||
+ (now.ConversionDeleted ? "; the conversion of its originals had been deleted" : string.Empty));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An entry as the trash lists it and the emptying plans it (<see cref="PlannedAsync"/>), from its record and folder as
|
||||
/// they are now; null when its record cannot be read.
|
||||
/// An entry as the trash lists it and an erase plans it, from its record and folder as they are now, and a replaced
|
||||
/// entry's survey against where its sequence was found under its lease (<paramref name="located"/>, which says whether the
|
||||
/// conversion of one of its photos was deleted, <see cref="TrashEntry.ConversionDeleted"/>); a null entry when its record
|
||||
/// cannot be read or its folder is gone.
|
||||
/// </summary>
|
||||
private async Task<TrashEntry?> RereadAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
|
||||
private static (TrashEntry? Entry, ConversionSurvey? Survey) Reread(Archive archive, ArchivePaths paths, TrashEntry entry, Located? located,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var (folder, recordPath) = Location(paths, entry);
|
||||
var now = FrameTrash.IsPhotoEntry(entry.Name)
|
||||
? FrameTrash.ReadListed(paths, entry.ArchiveId, entry.Name)
|
||||
: ReadRecord(recordPath) is { } record ? Entry(entry.ArchiveId, Path.GetFileName(entry.Name), record) : null;
|
||||
if (now is null) return null;
|
||||
if (now is null || !Directory.Exists(folder)) return (null, null);
|
||||
var (files, bytes) = Measure(folder, cancellationToken);
|
||||
return await PlannedAsync(paths, now with { Files = files, Bytes = bytes }, cancellationToken).ConfigureAwait(false);
|
||||
now = now with { Files = files, Bytes = bytes };
|
||||
if (located is null) return (now, null);
|
||||
if (FrameTrash.ReadEntry(paths, entry.Name) is not { } photos) return (null, null);
|
||||
var survey = Survey(archive, paths, entry.Name, photos, located, cancellationToken);
|
||||
return (survey.Deleted ? now with { ConversionDeleted = true } : now, survey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <paramref name="entry"/> as an emptying plans it: a replaced entry is <see cref="TrashEntry.ConversionDeleted"/> when the
|
||||
/// archive shows the conversion of one of its photos deleted (<see cref="Survey"/>), so the dry run says so. Its sequence
|
||||
/// is found once per plan (<paramref name="located"/>). Reads sidecars and records; hashes nothing.
|
||||
/// </summary>
|
||||
private async Task<TrashEntry> PlannedAsync(Archive archive, ArchivePaths paths, TrashEntry entry, Dictionary<Guid, Located> located,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry;
|
||||
if (!located.TryGetValue(entry.SequenceId, out var where))
|
||||
located[entry.SequenceId] = where = await LocateAsync(archive, paths, entry, null, cancellationToken).ConfigureAwait(false);
|
||||
return Survey(archive, paths, entry.Name, record, where, cancellationToken).Deleted ? entry with { ConversionDeleted = true } : entry;
|
||||
}
|
||||
|
||||
/// <summary>The folder and record of an entry of either kind.</summary>
|
||||
|
||||
@@ -5,101 +5,273 @@ using Chronos.Core.Model;
|
||||
namespace Chronos.Data.Archiving;
|
||||
|
||||
/// <summary>
|
||||
/// Where the replacements of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash: in
|
||||
/// the sequence (the replacement, or what replaced it in turn: a conversion of a conversion, followed through the sequence's
|
||||
/// Replaced tombstones), in the trash (the conversion deleted as photos or with its sequence, or the original of a later
|
||||
/// conversion), or nowhere (deleted and erased).
|
||||
/// Where the conversions of a replaced entry's originals are now (owner review 2, R7), for the emptying of the trash, as the
|
||||
/// archive itself shows it, never as this host's index alone has it (another host sharing the archive may have converted,
|
||||
/// moved or deleted since): the sequence's sidecar, found where the index or the entry says the sequence is, in the trash,
|
||||
/// or by walking the archive's folders, followed through its Replaced tombstones (a conversion of a conversion), and the
|
||||
/// files it names. A conversion counts as deleted only when the archive shows it: a Deleted tombstone at the end of the
|
||||
/// chain, a sidecar that holds none of it while every photo of its folder is a frame it hashed, the sequence in the trash,
|
||||
/// or no sequence of that id anywhere (and none in this index either). Whatever the archive cannot tell keeps the entry,
|
||||
/// and so does a frame whose file changed since its conversion.
|
||||
/// </summary>
|
||||
public sealed partial class SequenceTrash
|
||||
{
|
||||
/// <summary>Links a replacement chain may have at most (a conversion of a conversion…), as sync follows them.</summary>
|
||||
private const int ReplacementSteps = 8;
|
||||
|
||||
/// <summary>
|
||||
/// <paramref name="entry"/> as an emptying plans it: a replaced entry is <see cref="TrashEntry.ConversionDeleted"/> when
|
||||
/// the replacement of one of its photos is held by no frame of its sequence any more (the user deleted that photo, or the
|
||||
/// whole sequence), so the dry run says so. Reads the index and the entry's record; hashes nothing.
|
||||
/// </summary>
|
||||
private async Task<TrashEntry> PlannedAsync(ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
|
||||
/// <summary>Where a replaced entry's sequence is, as the archive shows it.</summary>
|
||||
private enum Whereabouts
|
||||
{
|
||||
if (entry.Kind != TrashKind.Replaced || FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return entry;
|
||||
var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
foreach (var photo in record.Photos)
|
||||
{
|
||||
if (photo.Replacement is { } replacement && await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is null)
|
||||
return entry with { ConversionDeleted = true };
|
||||
}
|
||||
return entry;
|
||||
/// <summary>In a folder of the archive, whose sidecar names it.</summary>
|
||||
Live,
|
||||
|
||||
/// <summary>Deleted: its folder is in the trash.</summary>
|
||||
Trashed,
|
||||
|
||||
/// <summary>Nowhere: deleted and erased.</summary>
|
||||
Gone,
|
||||
|
||||
/// <summary>The archive cannot tell (<see cref="Located.Reason"/>).</summary>
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// <summary>What a sidecar shows of a conversion (<see cref="SequenceHoldings.Classify"/>).</summary>
|
||||
private enum Holding
|
||||
{
|
||||
/// <summary>Frames of the sequence hold it, or what replaced it in turn.</summary>
|
||||
Held,
|
||||
|
||||
/// <summary>The sequence no longer holds it: deleted.</summary>
|
||||
Deleted,
|
||||
|
||||
/// <summary>Its frame is there with other content: its file changed since the conversion.</summary>
|
||||
Changed,
|
||||
|
||||
/// <summary>The sidecar does not say.</summary>
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether every photo of a replaced entry may be erased now, checked where its replacement is now. One its sequence holds
|
||||
/// (followed through later conversions) must hash to it there, re-read now: the originals are never erased while their
|
||||
/// conversion is damaged. One its sequence no longer holds was deleted, as the confirmed dry run said
|
||||
/// (<see cref="TrashEntry.ConversionDeleted"/>): a copy the trash still holds (the deleted photos, the originals of a later
|
||||
/// conversion, the deleted sequence) must hash to it too, and one erased since leaves nothing to keep the originals for.
|
||||
/// A replaced entry's sequence as found: live (its sidecar), in the trash (the sidecar of each deleted folder of it), gone,
|
||||
/// or unknown and why.
|
||||
/// </summary>
|
||||
private async Task<bool> ReplacementsHoldAsync(Archive archive, ArchivePaths paths, TrashEntry entry, CancellationToken cancellationToken)
|
||||
private sealed record Located(Whereabouts Where, SequenceHoldings? Live = null, IReadOnlyList<SequenceHoldings>? Trashed = null,
|
||||
TrashKeptReason Reason = TrashKeptReason.Unknown)
|
||||
{
|
||||
if (FrameTrash.ReadEntry(paths, entry.Name) is not { } record) return false;
|
||||
var links = await LinksAsync(record.SequenceId, cancellationToken).ConfigureAwait(false);
|
||||
public static Located Unknown(TrashKeptReason reason = TrashKeptReason.Unknown) => new(Whereabouts.Unknown, Reason: reason);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What an emptying needs to know of a replaced entry: whether the conversion of one of its photos was deleted (the dry
|
||||
/// run says so, <see cref="TrashEntry.ConversionDeleted"/>), the files each photo's conversion must still hash to one of
|
||||
/// (in the sequence, or the copies the trash holds), and why the entry stays when the archive cannot tell, or shows a
|
||||
/// conversion changed (<see cref="Kept"/>, null when neither).
|
||||
/// </summary>
|
||||
private sealed record ConversionSurvey(bool Deleted, IReadOnlyList<IReadOnlyList<(string Path, string Sha256)>> MustMatch, TrashKeptReason? Kept);
|
||||
|
||||
/// <summary>
|
||||
/// The conversions of <paramref name="record"/>'s photos (the entry <paramref name="name"/>), where the archive shows them
|
||||
/// now: in the sequence as <paramref name="located"/> found it (<see cref="LocateAsync"/>), else among the copies the
|
||||
/// trash holds. Reads the trash's records and sidecars; hashes nothing.
|
||||
/// </summary>
|
||||
private static ConversionSurvey Survey(Archive archive, ArchivePaths paths, string name, PhotoTrashRecord record, Located located,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (located.Where == Whereabouts.Unknown) return new ConversionSurvey(false, [], located.Reason);
|
||||
// Gone with its sequence, deleted or erased since.
|
||||
var deleted = located.Where != Whereabouts.Live;
|
||||
var mustMatch = new List<IReadOnlyList<(string Path, string Sha256)>>();
|
||||
TrashCopies? copies = null;
|
||||
foreach (var photo in record.Photos)
|
||||
{
|
||||
if (photo.Replacement is not { } replacement) return false;
|
||||
if (await InSequenceAsync(record.SequenceId, links, replacement, cancellationToken).ConfigureAwait(false) is { } held)
|
||||
if (photo.Replacement is not { } replacement) return new ConversionSurvey(deleted, [], TrashKeptReason.Unknown);
|
||||
if (located.Live is { } live)
|
||||
{
|
||||
if (!await AnyHashesAsync(held.Frames.Select(f => (paths.FromRelative(f.RelativePath), held.Sha256)), cancellationToken).ConfigureAwait(false))
|
||||
return false;
|
||||
continue;
|
||||
var (state, held) = live.Classify(photo.Frame.Id, replacement);
|
||||
if (KeptFor(state) is { } kept) return new ConversionSurvey(deleted, [], kept);
|
||||
if (state == Holding.Held)
|
||||
{
|
||||
mustMatch.Add(held);
|
||||
continue;
|
||||
}
|
||||
deleted = true;
|
||||
}
|
||||
copies ??= Copies(archive, paths, entry.Name, record.SequenceId, links, cancellationToken);
|
||||
else if (located.Trashed is { Count: > 0 } trashed)
|
||||
{
|
||||
// Its frames went with the sequence: a deleted folder's sidecar must say clearly what became of it.
|
||||
var states = trashed.Select(t => t.Classify(photo.Frame.Id, replacement).State).ToList();
|
||||
if (!states.Contains(Holding.Held) && states.Select(KeptFor).FirstOrDefault(r => r is not null) is { } kept)
|
||||
return new ConversionSurvey(deleted, [], kept);
|
||||
}
|
||||
// A copy the trash still holds (the deleted photos, the originals of a later conversion, the deleted sequence) must
|
||||
// match; with none left, nothing keeps the originals.
|
||||
copies ??= Copies(archive, paths, name, record.SequenceId, located, cancellationToken);
|
||||
var found = copies.Of(replacement);
|
||||
if (found.Count > 0 && !await AnyHashesAsync(found, cancellationToken).ConfigureAwait(false)) return false;
|
||||
if (found.Count > 0) mustMatch.Add(found);
|
||||
}
|
||||
return true;
|
||||
return new ConversionSurvey(deleted, mustMatch, null);
|
||||
}
|
||||
|
||||
/// <summary>The Replaced tombstones of a sequence in the index: each content and what replaced it (lower-case SHA-256).</summary>
|
||||
private async Task<Dictionary<string, string>> LinksAsync(Guid sequenceId, CancellationToken cancellationToken)
|
||||
/// <summary>Why an entry stays for what a sidecar shows of one of its conversions; null when that lets it go.</summary>
|
||||
private static TrashKeptReason? KeptFor(Holding state) => state switch
|
||||
{
|
||||
var links = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
foreach (var removal in await _db.FrameRemovals.ListBySequenceAsync(sequenceId, cancellationToken).ConfigureAwait(false))
|
||||
Holding.Changed => TrashKeptReason.Mismatch,
|
||||
Holding.Unknown => TrashKeptReason.Unknown,
|
||||
_ => null,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Where the sequence of replaced entry <paramref name="entry"/> is, as the archive shows it: in the folder this index has
|
||||
/// for it, or the one the entry recorded, when its sidecar names it there; else its deleted folders in the trash; else in
|
||||
/// any folder of the archive whose sidecar names it (another index moved it). Nowhere is gone only when this index has no
|
||||
/// row of it either: a row the archive does not bear out is a disagreement a scan settles. A sidecar that cannot be read
|
||||
/// where the sequence may be, or a folder of photos without one there, is unknown. <paramref name="heldEntryLease"/> (an
|
||||
/// erase, which holds the entry's lease and the sequence's): the live sidecar is read as its writers read it, what a crash
|
||||
/// left in it settled first (an undo whose completion waited for this entry's lease included), and one still unsettled
|
||||
/// (another entry busy) keeps the entry as busy.
|
||||
/// </summary>
|
||||
private async Task<Located> LocateAsync(Archive archive, ArchivePaths paths, TrashEntry entry, string? heldEntryLease, CancellationToken cancellationToken)
|
||||
{
|
||||
var id = entry.SequenceId;
|
||||
var row = await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false);
|
||||
if (row is not null && row.ArchiveId != archive.Id) row = null;
|
||||
foreach (var folder in new[] { row?.Folder, entry.Folder }.OfType<string>().Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
if (removal.Reason == RemovalReason.Replaced && removal.ReplacementSha256 is { Length: > 0 } replacement)
|
||||
links[removal.Sha256.ToLowerInvariant()] = replacement.ToLowerInvariant();
|
||||
var at = paths.Sequence(folder);
|
||||
switch (Holder(at, id))
|
||||
{
|
||||
case true:
|
||||
return Live(at, id, heldEntryLease);
|
||||
case null:
|
||||
Log.Warning("archive", $"{entry.Title}: {folder} may hold its sequence but its sidecar cannot be read; its replaced originals stay in the trash");
|
||||
return Located.Unknown();
|
||||
}
|
||||
}
|
||||
return links;
|
||||
if (Trashed(paths, id, cancellationToken) is not { } trashed) return Located.Unknown();
|
||||
if (trashed.Count > 0) return new Located(Whereabouts.Trashed, Trashed: trashed);
|
||||
foreach (var found in await Task.Run(() => ArchiveScanner.FindSequenceFolders(paths, cancellationToken), cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (found.Sidecar is null) continue;
|
||||
var at = paths.Sequence(found.RelativeFolder);
|
||||
var named = SidecarId(at.ManifestPath);
|
||||
if (named == id) return Live(at, id, heldEntryLease);
|
||||
if (named is null)
|
||||
{
|
||||
Log.Warning("archive", $"The sidecar of {found.RelativeFolder} cannot be read; replaced originals of {entry.Title} stay in the trash");
|
||||
return Located.Unknown();
|
||||
}
|
||||
}
|
||||
if (row is null) return new Located(Whereabouts.Gone);
|
||||
Log.Info("archive", $"{entry.Title} is in the index but not in the archive; its replaced originals stay in the trash until a scan");
|
||||
return Located.Unknown();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The frames of the sequence that hold <paramref name="replacement"/>, or what replaced it in turn (<paramref name="links"/>),
|
||||
/// and that content; null when no frame of the sequence holds any of them (it is not in the index, or was deleted).
|
||||
/// Whether <paramref name="folder"/> holds sequence <paramref name="id"/>: true when its sidecar names it, false when the
|
||||
/// folder is not there, holds nothing of a sequence, or its sidecar names another one; null when it cannot be told (a
|
||||
/// sidecar that cannot be read, or photos without one).
|
||||
/// </summary>
|
||||
private async Task<(string Sha256, IReadOnlyList<Frame> Frames)?> InSequenceAsync(Guid sequenceId, IReadOnlyDictionary<string, string> links,
|
||||
string replacement, CancellationToken cancellationToken)
|
||||
private static bool? Holder(SequencePaths folder, Guid id)
|
||||
{
|
||||
var sha = replacement.ToLowerInvariant();
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
for (var step = 0; step <= ReplacementSteps && seen.Add(sha); step++)
|
||||
if (!Directory.Exists(folder.Folder)) return false;
|
||||
if (!File.Exists(folder.ManifestPath)) return ArchiveScanner.ListPhotos(folder).Count > 0 ? null : false;
|
||||
return SidecarId(folder.ManifestPath) is { } named ? named == id : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The live sequence in <paramref name="folder"/> with its sidecar (<see cref="LocateAsync"/>); busy while a change to it is
|
||||
/// not settled or a file of it is held open, unknown when its sidecar cannot be read.
|
||||
/// </summary>
|
||||
private static Located Live(SequencePaths folder, Guid id, string? heldEntryLease)
|
||||
{
|
||||
try
|
||||
{
|
||||
var holders = (await _db.Frames.FindBySha256Async(sha, cancellationToken).ConfigureAwait(false)).Where(f => f.SequenceId == sequenceId).ToList();
|
||||
if (holders.Count > 0) return (sha, holders);
|
||||
if (!links.TryGetValue(sha, out var next)) break;
|
||||
sha = next;
|
||||
SequenceManifest manifest;
|
||||
if (heldEntryLease is not null)
|
||||
{
|
||||
manifest = SequenceWriter.ReadLocked(folder, id, heldEntryLease);
|
||||
FrameTrash.EnsureSettled(folder);
|
||||
FrameTrash.EnsureReplaced(folder);
|
||||
}
|
||||
else
|
||||
{
|
||||
manifest = ManifestStore.ReadSequence(folder.ManifestPath);
|
||||
if (manifest.Id != id) return Located.Unknown();
|
||||
}
|
||||
return new Located(Whereabouts.Live, Live: new SequenceHoldings(folder, manifest));
|
||||
}
|
||||
catch (LeaseUnavailableException ex)
|
||||
{
|
||||
Log.Info("archive", $"{folder.RelativeFolder} holds a change not settled yet; its replaced originals stay in the trash: {ex.Message}");
|
||||
return Located.Unknown(TrashKeptReason.Busy);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or UnauthorizedAccessException or FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
Log.Warning("archive", $"The sidecar of {folder.RelativeFolder} cannot be read; its replaced originals stay in the trash", ex);
|
||||
return Located.Unknown();
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
// Open elsewhere (a change being settled could not move a file yet): another try later.
|
||||
Log.Warning("archive", $"{folder.RelativeFolder} is in use; its replaced originals stay in the trash", ex);
|
||||
return Located.Unknown(TrashKeptReason.Busy);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The deleted folders of sequence <paramref name="id"/> in the trash, each with its sidecar; null when one of them has
|
||||
/// no sidecar that can be read (what it holds of the conversions cannot be told), or an entry's sequence cannot be told.
|
||||
/// </summary>
|
||||
private static List<SequenceHoldings>? Trashed(ArchivePaths paths, Guid id, CancellationToken cancellationToken)
|
||||
{
|
||||
var found = new List<SequenceHoldings>();
|
||||
var trash = TrashDirectory(paths);
|
||||
if (!Directory.Exists(trash)) return found;
|
||||
foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath));
|
||||
if (!Directory.Exists(folder)) continue;
|
||||
var deleted = paths.Sequence(paths.ToRelative(folder));
|
||||
var owner = ReadRecord(recordPath)?.Sequence.Id ?? (File.Exists(deleted.ManifestPath) ? SidecarId(deleted.ManifestPath) : null);
|
||||
if (owner is null)
|
||||
{
|
||||
Log.Warning("archive", $"Trash entry {Path.GetFileName(folder)} cannot be told apart; replaced originals of its sequence stay in the trash");
|
||||
return null;
|
||||
}
|
||||
if (owner != id) continue;
|
||||
try
|
||||
{
|
||||
found.Add(new SequenceHoldings(deleted, ManifestStore.ReadSequence(deleted.ManifestPath)));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; the replaced originals of its photos stay in the trash", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
/// <summary>Adds the Replaced tombstones of a sidecar to <paramref name="copies"/>: each content and what replaced it.</summary>
|
||||
private static void AddLinks(TrashCopies copies, SequenceManifest manifest)
|
||||
{
|
||||
foreach (var removed in manifest.Removed ?? [])
|
||||
{
|
||||
if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { Length: > 0 } replacement) copies.Link(removed.Sha256, replacement);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the trash holds of a sequence's photos, but the entry <paramref name="except"/> (the one being erased): the files of
|
||||
/// its photo entries (deleted photos, replaced originals) and of its own entry when the whole sequence is in the trash, by
|
||||
/// content, and the replacements those entries name. Read from their records and sidecars; an unreadable one adds nothing.
|
||||
/// its photo entries (deleted photos, replaced originals) and of its deleted folders (<paramref name="located"/>), by
|
||||
/// content, and the replacements those entries and the sequence's sidecar name. Read from their records and sidecars; an
|
||||
/// unreadable record adds nothing.
|
||||
/// </summary>
|
||||
private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, IReadOnlyDictionary<string, string> links,
|
||||
CancellationToken cancellationToken)
|
||||
private static TrashCopies Copies(Archive archive, ArchivePaths paths, string except, Guid sequenceId, Located located, CancellationToken cancellationToken)
|
||||
{
|
||||
var copies = new TrashCopies(links);
|
||||
var copies = new TrashCopies();
|
||||
if (located.Live is { } live) AddLinks(copies, live.Manifest);
|
||||
foreach (var listed in FrameTrash.List(archive, cancellationToken).Where(e => e.SequenceId == sequenceId && e.Name != except))
|
||||
{
|
||||
if (FrameTrash.ReadEntry(paths, listed.Name) is not { } record) continue;
|
||||
@@ -110,33 +282,24 @@ public sealed partial class SequenceTrash
|
||||
if (photo.Frame.Sha256 is { } original && photo.Replacement is { } replacement) copies.Link(original, replacement);
|
||||
}
|
||||
}
|
||||
var trash = TrashDirectory(paths);
|
||||
if (!Directory.Exists(trash)) return copies;
|
||||
foreach (var recordPath in Directory.EnumerateFiles(trash, "*" + RecordExtension))
|
||||
foreach (var deleted in located.Trashed ?? [])
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var folder = Path.Combine(trash, Path.GetFileNameWithoutExtension(recordPath));
|
||||
if (!Directory.Exists(folder) || ReadRecord(recordPath)?.Sequence.Id != sequenceId) continue;
|
||||
var deleted = paths.Sequence(paths.ToRelative(folder));
|
||||
SequenceManifest manifest;
|
||||
try
|
||||
{
|
||||
manifest = ManifestStore.ReadSequence(deleted.ManifestPath);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("archive", $"The sidecar of trash entry {Path.GetFileName(folder)} cannot be read; its photos are not looked at", ex);
|
||||
continue;
|
||||
}
|
||||
foreach (var frame in manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.File(frame.File));
|
||||
foreach (var removed in manifest.Removed ?? [])
|
||||
{
|
||||
if (removed.Reason == RemovalReason.Replaced && removed.Replacement is { } replacement) copies.Link(removed.Sha256, replacement);
|
||||
}
|
||||
foreach (var frame in deleted.Manifest.Frames.Where(f => f.Sha256 is not null)) copies.Add(frame.Sha256!, deleted.Folder.File(frame.File));
|
||||
AddLinks(copies, deleted.Manifest);
|
||||
}
|
||||
return copies;
|
||||
}
|
||||
|
||||
/// <summary>Whether, for each photo of a survey, one of the files its conversion must match is there and hashes to it now.</summary>
|
||||
private static async Task<bool> MatchAsync(ConversionSurvey survey, CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (var files in survey.MustMatch)
|
||||
{
|
||||
if (!await AnyHashesAsync(files, cancellationToken).ConfigureAwait(false)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Whether one of <paramref name="files"/> is there and hashes to its SHA-256 now.</summary>
|
||||
private static async Task<bool> AnyHashesAsync(IEnumerable<(string Path, string Sha256)> files, CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -148,10 +311,87 @@ public sealed partial class SequenceTrash
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>Copies of a sequence's photos in the trash by content, and the replacement links known (<see cref="Copies"/>).</summary>
|
||||
private sealed class TrashCopies(IReadOnlyDictionary<string, string> links)
|
||||
/// <summary>
|
||||
/// A sequence's sidecar (live, or of a deleted folder) as a survey reads it, indexed once, so an entry of a conversion of
|
||||
/// tens of thousands of photos is weighed in one pass over it: its frames by content and by id, its tombstones by content,
|
||||
/// whether every frame is hashed, and (read when first needed) whether its folder holds a photo the sidecar does not list.
|
||||
/// </summary>
|
||||
private sealed class SequenceHoldings
|
||||
{
|
||||
private readonly Dictionary<string, string> _links = new(links, StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, List<SidecarFrame>> _byContent = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<Guid, SidecarFrame> _byId = [];
|
||||
private readonly Dictionary<string, RemovedPhoto> _tombstones = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly bool _unhashed;
|
||||
private bool? _unlisted;
|
||||
|
||||
public SequenceHoldings(SequencePaths folder, SequenceManifest manifest)
|
||||
{
|
||||
Folder = folder;
|
||||
Manifest = manifest;
|
||||
foreach (var frame in manifest.Frames)
|
||||
{
|
||||
_byId.TryAdd(frame.Id, frame);
|
||||
if (frame.Sha256 is not { } sha)
|
||||
{
|
||||
_unhashed = true;
|
||||
continue;
|
||||
}
|
||||
if (!_byContent.TryGetValue(sha, out var holders)) _byContent[sha] = holders = [];
|
||||
holders.Add(frame);
|
||||
}
|
||||
foreach (var tombstone in manifest.Removed ?? []) _tombstones[tombstone.Sha256] = tombstone;
|
||||
}
|
||||
|
||||
public SequencePaths Folder { get; }
|
||||
|
||||
public SequenceManifest Manifest { get; }
|
||||
|
||||
/// <summary>
|
||||
/// What the sidecar shows of <paramref name="replacement"/>, the conversion that took the place of frame
|
||||
/// <paramref name="frameId"/>'s photo, followed through its Replaced tombstones: the frames holding it, each file with
|
||||
/// the content it must hash to; changed, when the frame is there with other content; deleted, when a Deleted tombstone
|
||||
/// ends the chain, or none does and the frame is gone while every photo of the folder is a frame the sidecar hashed;
|
||||
/// unknown otherwise (the frame there and never hashed again, a frame never hashed or a photo the sidecar does not list
|
||||
/// that may hold it, a chain too long or looping).
|
||||
/// </summary>
|
||||
public (Holding State, List<(string Path, string Sha256)> Files) Classify(Guid frameId, string replacement)
|
||||
{
|
||||
var chain = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
RemovedPhoto? end = null;
|
||||
for (var sha = replacement.ToLowerInvariant(); ;)
|
||||
{
|
||||
if (chain.Count > ReplacementSteps || !chain.Add(sha)) return (Holding.Unknown, []);
|
||||
if (_byContent.TryGetValue(sha, out var holders))
|
||||
{
|
||||
var content = sha;
|
||||
return (Holding.Held, [.. holders.Select(f => (Folder.File(f.File), content))]);
|
||||
}
|
||||
if (!_tombstones.TryGetValue(sha, out end) || end.Reason != RemovalReason.Replaced) break;
|
||||
if (end.Replacement is not { Length: > 0 } next) return (Holding.Unknown, []);
|
||||
sha = next.ToLowerInvariant();
|
||||
}
|
||||
if (_byId.TryGetValue(frameId, out var frame)) return (frame.Sha256 is null ? Holding.Unknown : Holding.Changed, []);
|
||||
if (end is { Reason: RemovalReason.Deleted }) return (Holding.Deleted, []);
|
||||
return _unhashed || Unlisted() ? (Holding.Unknown, []) : (Holding.Deleted, []);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the folder holds a photo the sidecar does not list, compared as a scan finds the photos it adopts
|
||||
/// (<see cref="SequenceAdopter.FindOrphans"/>), read once.
|
||||
/// </summary>
|
||||
private bool Unlisted()
|
||||
{
|
||||
if (_unlisted is { } known) return known;
|
||||
var listed = Manifest.Frames.Select(f => f.File).ToHashSet(StringComparer.Ordinal);
|
||||
_unlisted = ArchiveScanner.ListPhotos(Folder).Any(p => !listed.Contains(p));
|
||||
return _unlisted.Value;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Copies of a sequence's photos in the trash by content, and the replacement links known (<see cref="Copies"/>).</summary>
|
||||
private sealed class TrashCopies
|
||||
{
|
||||
private readonly Dictionary<string, string> _links = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, List<string>> _files = new(StringComparer.Ordinal);
|
||||
|
||||
public void Add(string sha256, string path)
|
||||
|
||||
@@ -74,9 +74,10 @@ public sealed record TrashEntry(Guid ArchiveId, string Name, Guid SequenceId, st
|
||||
public Guid? JobId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// A <see cref="TrashKind.Replaced"/> entry whose conversion was deleted since: a photo's replacement (followed through
|
||||
/// later conversions) is no longer in its sequence, so erasing the entry may erase the last copy of that photo. Set by the
|
||||
/// dry run of an emptying (<see cref="SequenceTrash.PlanEmptyAsync"/>), which says so; false in a plain listing.
|
||||
/// A <see cref="TrashKind.Replaced"/> entry whose conversion was deleted since: the archive shows a photo's replacement
|
||||
/// (followed through later conversions) deleted from its sequence, or the sequence itself deleted, so erasing the entry may
|
||||
/// erase the last copy of that photo. Set by the dry run of an emptying (<see cref="SequenceTrash.PlanEmptyAsync"/>), which
|
||||
/// says so; false in a plain listing.
|
||||
/// </summary>
|
||||
public bool ConversionDeleted { get; init; }
|
||||
}
|
||||
|
||||
@@ -280,7 +280,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
// the converted file that replaced it) is not filed again, which would also drop its tombstone.
|
||||
File.Delete(staged);
|
||||
Interlocked.Increment(ref refused);
|
||||
if (tombstones.DeletedLocally(sha)) context.Log($"{frame.FileName}: deleted on this PC, not downloaded");
|
||||
if (tombstones.DeletedLocally(sha)) context.Log(Loc.L("Archive.Log.SyncDeletedHere", frame.FileName));
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -299,11 +299,13 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
|
||||
if (created) await sync.TakeAllAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
await sync.RecompareAsync(remote, remoteId, ct).ConfigureAwait(false);
|
||||
// Originals filed into the sequence (not restores, nor frames swapped to another encoding, whose passes are queued as
|
||||
// they swap) need their thumbnails and light analysis.
|
||||
if (sequenceId is { } downloaded && wanted.Count > restores.Count + aliases.Count + refused)
|
||||
{
|
||||
var label = local!.DisplayTitle;
|
||||
await EnqueueFollowUpAsync(context, ArchiveJobs.Thumbnails([downloaded], label)).ConfigureAwait(false);
|
||||
await EnqueueFollowUpAsync(context, ProcessingJobs.LightAnalysis([downloaded], label)).ConfigureAwait(false);
|
||||
await QueuePassesAsync(context, downloaded, [ArchiveJobs.Thumbnails([downloaded], label), ProcessingJobs.LightAnalysis([downloaded], label)])
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
if (moved) throw new InvalidOperationException(Loc.L("Errors.Archive.FolderMoved"));
|
||||
}
|
||||
@@ -374,24 +376,26 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
/// Gives each frame flagged missing here whose photo the server holds only in another encoding (a conversion on either
|
||||
/// side, owner review 2, R7) the verified copy fetched (<see cref="FrameTrash.ReplaceMissingAsync"/>: under the sequence's
|
||||
/// lease, in its folder as the index has it then, whose sidecar must be the sequence's, <see cref="SequenceWriter.ReadLocked"/>).
|
||||
/// Run after the transfers, once <see cref="RestoreAsync"/> found the sequence in that folder. Returns how many took their
|
||||
/// frame's place; the copy of a frame that changed meanwhile, or whose file came back, is dropped (logged). Moved, with the
|
||||
/// copies not swapped yet kept in staging for the next attempt, when the sequence left that folder meanwhile.
|
||||
/// Run after the transfers, once <see cref="RestoreAsync"/> found the sequence in that folder. A swap clears what covered
|
||||
/// the sequence, as any replacement does, so the passes it had (<see cref="FrameTrash.FollowUps"/>: the light analysis, a
|
||||
/// background deep analysis, the thumbnails and proxies) are queued after each one, whatever stops the download later.
|
||||
/// Returns how many took their frame's place; the copy of a frame that changed meanwhile, or whose file came back, is
|
||||
/// dropped (logged). Moved, with the copies not swapped yet kept in staging for the next attempt, when the sequence left
|
||||
/// that folder meanwhile.
|
||||
/// </summary>
|
||||
private async Task<(int Swapped, bool Moved)> SwapAliasesAsync(JobContext context, Archive archive, Sequence local, ArchivePaths paths,
|
||||
IEnumerable<AliasCopy> aliases, CancellationToken ct)
|
||||
{
|
||||
var swapped = 0;
|
||||
// What the sequence had before the first swap cleared it: the passes that make it again.
|
||||
var had = SequenceStatus.None;
|
||||
foreach (var alias in aliases)
|
||||
{
|
||||
ReplaceResult? result;
|
||||
try
|
||||
{
|
||||
if (await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256,
|
||||
alias.FileName, ct).ConfigureAwait(false))
|
||||
{
|
||||
swapped++;
|
||||
continue;
|
||||
}
|
||||
result = await sync.Archives.FrameTrash.ReplaceMissingAsync(archive, local.Id, alias.Lost.Id, alias.Lost.Sha256!, alias.Staged, alias.Sha256,
|
||||
alias.FileName, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or DirectoryNotFoundException or FileNotFoundException)
|
||||
{
|
||||
@@ -400,12 +404,32 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
throw;
|
||||
return (swapped, true);
|
||||
}
|
||||
File.Delete(alias.Staged);
|
||||
context.Log($"{alias.FileName}: its frame here changed meanwhile, not downloaded");
|
||||
if (result is null)
|
||||
{
|
||||
File.Delete(alias.Staged);
|
||||
context.Log(Loc.L("Archive.Log.SyncFrameChanged", alias.FileName));
|
||||
continue;
|
||||
}
|
||||
swapped++;
|
||||
had |= result.Had;
|
||||
await QueuePassesAsync(context, local.Id, sync.Archives.FrameTrash.FollowUps(archive, result.Sequence, had)).ConfigureAwait(false);
|
||||
}
|
||||
return (swapped, false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues <paramref name="requests"/>, passes over the sequence <paramref name="sequenceId"/>, as follow-ups of this job;
|
||||
/// one already queued for the sequence (not started yet, so it covers what changed) is left out.
|
||||
/// </summary>
|
||||
private async Task QueuePassesAsync(JobContext context, Guid sequenceId, IEnumerable<ArchiveJobRequest> requests)
|
||||
{
|
||||
foreach (var request in requests)
|
||||
{
|
||||
if (await sync.Database.Jobs.IsQueuedAsync(request.Kind, sequenceId, CancellationToken.None).ConfigureAwait(false)) continue;
|
||||
await EnqueueFollowUpAsync(context, request).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A verified copy fetched in another encoding of a frame flagged missing here (<see cref="SwapAliasesAsync"/>).</summary>
|
||||
private sealed record AliasCopy(string Staged, string Sha256, Frame Lost, string FileName);
|
||||
|
||||
@@ -578,7 +602,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
var cached = files.Count(f => f is not null);
|
||||
var standIns = Enumerable.Range(0, frames.Count).Count(i => kinds[i] != ProxyCache.Proxy && !NeverProxied(frames[i]));
|
||||
await sync.SetProxiesAsync(remote.Server, dto, cached, standIns, manifest.FramesVersion, ct).ConfigureAwait(false);
|
||||
if (cached < frames.Count) context.Log($"{frames.Count - cached} frame(s) have neither proxy nor thumbnail on the server");
|
||||
if (cached < frames.Count) context.Log(Loc.Plural("Archive.Log.SyncNoPreview", frames.Count - cached));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -628,7 +652,7 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
context.Log($"{frame.FileName}: missing, not uploaded");
|
||||
context.Log(Loc.L("Archive.Log.SyncMissing", frame.FileName));
|
||||
return null;
|
||||
}
|
||||
var sha = await ChronosClient.HashFileAsync(path, cancellationToken).ConfigureAwait(false);
|
||||
@@ -638,13 +662,13 @@ internal sealed class SyncTransfers(SyncService sync)
|
||||
hashes[frame.Id] = sha;
|
||||
if (tombstones.DeletedRemotely(sha))
|
||||
{
|
||||
context.Log($"{frame.FileName}: deleted on the server, not uploaded");
|
||||
context.Log(Loc.L("Archive.Log.SyncDeletedThere", frame.FileName));
|
||||
return notSent;
|
||||
}
|
||||
}
|
||||
if (frame.Sha256 is { } archived && !archived.Equals(sha, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
context.Log($"{frame.FileName}: changed since it was archived, not uploaded");
|
||||
context.Log(Loc.L("Archive.Log.SyncChanged", frame.FileName));
|
||||
return null;
|
||||
}
|
||||
return await remote.UploadFileAsync(path, sequenceId, sessionKey, sha, frame.CapturedAt, cancellationToken).ConfigureAwait(false) ?? notSent;
|
||||
|
||||
@@ -136,8 +136,9 @@ export async function renderTrash(root, { api, setMeta, actions, onEvent }, { ba
|
||||
const confirmed = await confirmDialog({
|
||||
title: t("Admin.Trash.EmptyTitle"),
|
||||
body: [plural("Admin.Trash.EmptyBody", plan.files, formatBytes(plan.bytes)), t("Admin.Trash.EmptyFinal")].join(" "),
|
||||
// Replaced originals whose conversion was deleted since may be the last copy of their photos: the list says so.
|
||||
list: plan.entries.map((e) => t("Admin.Trash.PlanItem",
|
||||
// Replaced originals whose conversion was deleted since may be the last copy of their photos: the list names them
|
||||
// first, each saying so, wherever the trash lists them.
|
||||
list: [...plan.entries.filter((e) => e.conversionDeleted), ...plan.entries.filter((e) => !e.conversionDeleted)].map((e) => t("Admin.Trash.PlanItem",
|
||||
e.conversionDeleted ? `${kindLabel(e.kind)} · ${t("Admin.Trash.ConversionDeleted")}` : kindLabel(e.kind), e.title, formatBytes(e.bytes))),
|
||||
confirm: t("Admin.Trash.EmptyConfirm"),
|
||||
confirmTip: t("Admin.Trash.EmptyConfirm.Tip"),
|
||||
|
||||
@@ -70,6 +70,38 @@ public sealed class TrashDialogTests(TestContext context)
|
||||
+ Loc.L("Archive.Trash.ConversionDeleted"), lone.Detail!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryEntryThatMayHoldTheLastCopyIsNamedBeforeTheTypedConfirmation()
|
||||
{
|
||||
Loc.SetCulture(CultureInfo.GetCultureInfo("en"));
|
||||
// The trash lists newest first: ten deletions since two old conversions whose converted photos were deleted.
|
||||
var newer = Enumerable.Range(1, 10).Select(i => Entry($"Deleted {i}", TrashKind.Photos, 1, 1_000_000)).ToList();
|
||||
var oldest = new[]
|
||||
{
|
||||
Entry("Etna", TrashKind.Replaced, 3, 90_000_000) with { ConversionDeleted = true },
|
||||
Entry("Stromboli", TrashKind.Replaced, 2, 60_000_000) with { ConversionDeleted = true },
|
||||
};
|
||||
var plan = new TrashEmptyPlan(Guid.CreateVersion7(), [.. newer, .. oldest], 160_000_000, "f1", []);
|
||||
|
||||
var lines = Assert.NotNull(TrashEmptyPlans.Request(plan)).Detail!.Split(Environment.NewLine);
|
||||
|
||||
var why = Loc.Separator + Loc.L("Archive.Trash.ConversionDeleted");
|
||||
Assert.True(lines[0].Contains("Etna", StringComparison.Ordinal) && lines[0].Contains(why, StringComparison.Ordinal), lines[0]);
|
||||
Assert.True(lines[1].Contains("Stromboli", StringComparison.Ordinal) && lines[1].Contains(why, StringComparison.Ordinal), lines[1]);
|
||||
// The other entries fill the lines left, and the rest are counted.
|
||||
Assert.Equal(DeletePlans.ListedLines + 1, lines.Length);
|
||||
Assert.Equal(Loc.Plural("Archive.Delete.More", newer.Count - (DeletePlans.ListedLines - oldest.Length)), lines[^1]);
|
||||
|
||||
// More of them than lines: every one is named all the same.
|
||||
var many = Enumerable.Range(1, DeletePlans.ListedLines + 3).Select(i => Entry($"Converted {i}", TrashKind.Replaced, 1, 1_000_000) with { ConversionDeleted = true })
|
||||
.ToList();
|
||||
lines = Assert.NotNull(TrashEmptyPlans.Request(plan with { Entries = [.. newer, .. many] })).Detail!.Split(Environment.NewLine);
|
||||
Assert.Equal(many.Count + 1, lines.Length);
|
||||
for (var i = 0; i < many.Count; i++)
|
||||
Assert.True(lines[i].StartsWith(many[i].Title + " ", StringComparison.Ordinal) && lines[i].Contains(why, StringComparison.Ordinal), lines[i]);
|
||||
Assert.Equal(Loc.Plural("Archive.Delete.More", newer.Count), lines[^1]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task EmptyQueuesTheJobOfTheConfirmedPlanOnly()
|
||||
{
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Security.Cryptography;
|
||||
using Chronos.Core.Archiving;
|
||||
using Chronos.Core.Import;
|
||||
using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Data.Archiving;
|
||||
@@ -45,9 +46,11 @@ public sealed class ReplacedTrashTests(TestContext context)
|
||||
|
||||
/// <summary>
|
||||
/// Replaces the files of the frames at <paramref name="indices"/> (in the sidecar now) with other photos, as a
|
||||
/// conversion's stage B does (<see cref="FrameTrash.ReplaceAsync"/>); returns the trash entry of the originals.
|
||||
/// conversion's stage B does (<see cref="FrameTrash.ReplaceAsync"/>), on this host or on <paramref name="by"/> (another
|
||||
/// host sharing the archive, whose index this one does not read); returns the trash entry of the originals.
|
||||
/// </summary>
|
||||
private static async Task<TrashEntry> ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList<int> indices, int seed, Guid? job = null)
|
||||
private static async Task<TrashEntry> ReplaceAsync(ArchiveFixture fx, Guid sequenceId, IReadOnlyList<int> indices, int seed, Guid? job = null,
|
||||
(ArchiveService Service, Archive Archive)? by = null)
|
||||
{
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(sequenceId))!;
|
||||
var manifest = Sidecar(fx, sequence);
|
||||
@@ -68,10 +71,21 @@ public sealed class ReplacedTrashTests(TestContext context)
|
||||
Source = new FrameSource { Sha256 = current.Sha256, Format = current.Format, File = current.File, Size = current.Size },
|
||||
}));
|
||||
}
|
||||
var result = Assert.NotNull(await fx.Service.FrameTrash.ReplaceAsync(fx.Archive, sequenceId, null, swaps, null, job, null, null, fx.Ct));
|
||||
var (service, archive) = by ?? (fx.Service, fx.Archive);
|
||||
var result = Assert.NotNull(await service.FrameTrash.ReplaceAsync(archive, sequenceId, null, swaps, null, job, null, null, fx.Ct));
|
||||
return Assert.NotNull(result.Entry);
|
||||
}
|
||||
|
||||
/// <summary>Flips a byte near the end of <paramref name="path"/>; returns its bytes before.</summary>
|
||||
private static byte[] Damage(string path)
|
||||
{
|
||||
var bytes = File.ReadAllBytes(path);
|
||||
var damaged = (byte[])bytes.Clone();
|
||||
damaged[^10] ^= 0x01;
|
||||
File.WriteAllBytes(path, damaged);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
private static SequenceManifest Sidecar(ArchiveFixture fx, Sequence sequence) => ManifestStore.ReadSequence(fx.Paths.Sequence(sequence.Folder).ManifestPath);
|
||||
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
@@ -279,6 +293,213 @@ public sealed class ReplacedTrashTests(TestContext context)
|
||||
Assert.Count(2, await fx.Db.Frames.ListBySequenceAsync(id));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AConversionAnotherHostMadeIsHashedWhereTheArchiveHoldsIt()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (id, photos) = await UploadedAsync(fx, 2);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
// A desktop sharing the archive converts the first photo; this index still holds the original there.
|
||||
var replaced = await ReplaceAsync(fx, id, [0], 500, by: await fx.OtherWriterAsync());
|
||||
Assert.Equal(Sha(photos[0]), (await fx.Db.Frames.ListBySequenceAsync(id))[0].Sha256);
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
||||
var converted = fx.Paths.Sequence(sequence.Folder).File(Sidecar(fx, sequence).Frames[0].File);
|
||||
|
||||
// The conversion is damaged since: the dry run finds it in the sequence, and the emptying keeps the originals.
|
||||
var bytes = Damage(converted);
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.False(plan.Entries.Single().ConversionDeleted, "the archive holds the conversion, whatever this index has");
|
||||
var kept = await EmptyAsync(fx, engine, plan, JobState.Failed);
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail);
|
||||
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
||||
|
||||
// Whole again: the originals go.
|
||||
File.WriteAllBytes(converted, bytes);
|
||||
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done);
|
||||
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AConversionAnotherIndexMovedIsHashedWhereItIsNow()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (id, _) = await UploadedAsync(fx, 2);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
||||
// Another index moves the sequence into another folder; this one has not scanned since.
|
||||
var before = (await fx.Db.Sequences.GetAsync(id))!.Folder;
|
||||
var (other, archive) = await fx.OtherWriterAsync();
|
||||
await other.ApplyRenameAsync(await other.PlanRenameAsync(archive.Id, [id], "moved/{title}", null, fx.Ct), null, fx.Ct);
|
||||
Assert.Equal(before, (await fx.Db.Sequences.GetAsync(id))!.Folder);
|
||||
var folder = ArchiveScanner.FindSequenceFolders(fx.Paths, fx.Ct).Select(s => fx.Paths.Sequence(s.RelativeFolder)).Single(s => s.RelativeFolder != before);
|
||||
Assert.True(folder.RelativeFolder.StartsWith("moved", StringComparison.Ordinal), folder.RelativeFolder);
|
||||
var converted = folder.File(ManifestStore.ReadSequence(folder.ManifestPath).Frames[0].File);
|
||||
|
||||
var bytes = Damage(converted);
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.False(plan.Entries.Single().ConversionDeleted, "found in the folder the sequence is in now");
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
||||
|
||||
File.WriteAllBytes(converted, bytes);
|
||||
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Done);
|
||||
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OriginalsStayWhileTheArchiveCannotTellWhereTheirConversionIs()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (id, _) = await UploadedAsync(fx, 2);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var manifest = Sidecar(fx, sequence);
|
||||
var unknown = Loc.L("Archive.Log.TrashKeptUnknown", replaced.Title);
|
||||
|
||||
// A sidecar rebuilt from the photos, as a scan adopts a folder that lost its own: new frame ids, nothing hashed, no
|
||||
// tombstones. It no longer says what became of the conversion.
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||||
{
|
||||
Frames = [.. manifest.Frames.Select(f => f with { Id = Guid.CreateVersion7(), Sha256 = null })],
|
||||
Removed = null,
|
||||
});
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.False(plan.Entries.Single().ConversionDeleted);
|
||||
Assert.Contains(unknown, (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
||||
|
||||
// A sidecar that cannot be read.
|
||||
File.WriteAllText(folder.ManifestPath, "{ \"damaged");
|
||||
Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail);
|
||||
|
||||
// The sequence gone from the archive (deleted and erased by another host) while this index still has it: kept until a
|
||||
// scan settles the index. Then the dry run says the conversion was deleted, and the emptying confirmed with it erases.
|
||||
Directory.Delete(folder.Folder, recursive: true);
|
||||
Assert.Contains(unknown, (await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct), JobState.Failed)).LogTail);
|
||||
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
||||
await fx.ScanAsync(ScanMode.Rebuild);
|
||||
Assert.Null(await fx.Db.Sequences.GetAsync(id));
|
||||
var gone = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.True(gone.Entries.Single().ConversionDeleted, "no sequence of that id anywhere");
|
||||
await EmptyAsync(fx, engine, gone, JobState.Done);
|
||||
Assert.Empty(await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OriginalsOfABusySequenceStayInTheTrash()
|
||||
{
|
||||
await using var fx = await CreateAsync(context, configure: o => o with { LeaseWait = TimeSpan.FromMilliseconds(300) });
|
||||
var (id, _) = await UploadedAsync(fx, 2);
|
||||
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
var parameters = new EmptyTrashParameters([.. plan.Entries.Select(e => new EmptyTrashItem(e.Name, SequenceTrash.EntryFingerprint(e)))], plan.Fingerprint);
|
||||
|
||||
// Another host writes the sequence meanwhile: the conversion is not weighed under it.
|
||||
using (Assert.NotNull(LeaseFile.TryAcquire(fx.Paths.LocksDirectory, $"seq-{id:N}", "another host")))
|
||||
{
|
||||
var busy = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((0, TrashKeptReason.Busy), (busy.Erased, busy.Refused.Single().Reason));
|
||||
Assert.Equal(Loc.L("Archive.Log.TrashKeptBusy", replaced.Title), busy.Refused.Single().LogLine);
|
||||
}
|
||||
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
||||
|
||||
var free = await fx.Service.Trash.EmptyAsync(fx.Archive, parameters, 0, (_, _) => ValueTask.CompletedTask, fx.Ct);
|
||||
Assert.Equal((1, 0), (free.Erased, free.Refused.Count));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AConversionAnotherHostDeletedIsReadFromTheArchiveNotThisIndex()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (id, _) = await UploadedAsync(fx, 3);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
||||
var frames = await fx.Db.Frames.ListBySequenceAsync(id);
|
||||
// A desktop sharing the archive deletes the converted photo; this index still holds it.
|
||||
var (other, _) = await fx.OtherWriterAsync();
|
||||
var deletion = await other.FrameTrash.DeleteAsync(await other.FrameTrash.PlanDeleteAsync(id, [frames[0].Id], null, fx.Ct), null, null, fx.Ct);
|
||||
Assert.Count(3, await fx.Db.Frames.ListBySequenceAsync(id));
|
||||
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.True(plan.Entries.Single(e => e.Name == replaced.Name).ConversionDeleted, "the sidecar's tombstone says so, whatever this index holds");
|
||||
Assert.False(plan.Entries.Single(e => e.Name == deletion.Entry.Name).ConversionDeleted);
|
||||
|
||||
// The deleted conversion is in the trash, damaged: the originals stay.
|
||||
var record = Assert.NotNull(FrameTrash.ReadEntry(fx.Paths, deletion.Entry.Name));
|
||||
var copy = Path.Combine(FrameTrash.Location(fx.Paths, deletion.Entry.Name).Folder, record.Photos[0].Files[0].Path);
|
||||
var bytes = Damage(copy);
|
||||
var kept = await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Failed);
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), kept.LogTail);
|
||||
|
||||
// Whole again: the originals go, the deleted conversion stays where it is.
|
||||
File.WriteAllBytes(copy, bytes);
|
||||
await EmptyAsync(fx, engine, await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct), JobState.Done);
|
||||
Assert.Equal(deletion.Entry.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task OriginalsStayWhenTheirConversionChangedSince()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (id, _) = await UploadedAsync(fx, 2);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var manifest = Sidecar(fx, sequence);
|
||||
// The converted file was edited in place and hashed again: its frame holds other content, and nothing says what
|
||||
// became of the conversion.
|
||||
var edited = Photos.Jpeg(manifest.Frames[0].CapturedAt, 700);
|
||||
File.WriteAllBytes(folder.File(manifest.Frames[0].File), edited);
|
||||
ManifestStore.WriteSequence(folder.ManifestPath, manifest with
|
||||
{
|
||||
Frames = [.. manifest.Frames.Select((f, i) => i == 0 ? f with { Sha256 = Sha(edited), Size = edited.Length } : f)],
|
||||
});
|
||||
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, null, fx.Ct);
|
||||
Assert.False(plan.Entries.Single().ConversionDeleted, "the frame is there");
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKeptMismatch", replaced.Title), (await EmptyAsync(fx, engine, plan, JobState.Failed)).LogTail);
|
||||
Assert.Equal(replaced.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnEmptyingReadsAnEntryOnlyOnceAnUndoWaitingForItIsSettled()
|
||||
{
|
||||
await using var fx = await CreateAsync(context);
|
||||
var (id, photos) = await UploadedAsync(fx, 2);
|
||||
var engine = await fx.StartEngineAsync();
|
||||
var replaced = await ReplaceAsync(fx, id, [0], 500);
|
||||
var (entryFolder, recordPath) = FrameTrash.Location(fx.Paths, replaced.Name);
|
||||
var record = await File.ReadAllTextAsync(recordPath, fx.Ct);
|
||||
// Undone: the original is back in its frame, the conversion in an entry of its own.
|
||||
await fx.Service.RestoreEntryAsync(fx.Archive.Id, replaced.Name, null, fx.Ct);
|
||||
var undo = (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single();
|
||||
// As a crash right after the undo's sidecar was written leaves it: the entry its original came from still lists it,
|
||||
// and the undo's completion waits for that entry's lease.
|
||||
Directory.CreateDirectory(entryFolder);
|
||||
await File.WriteAllTextAsync(recordPath, record, fx.Ct);
|
||||
var sequence = (await fx.Db.Sequences.GetAsync(id))!;
|
||||
var folder = fx.Paths.Sequence(sequence.Folder);
|
||||
var manifest = Sidecar(fx, sequence);
|
||||
var (undoFolder, undoRecord) = FrameTrash.Location(fx.Paths, undo.Name);
|
||||
new FrameReplaceJournal(manifest.Versions.Frames - 1, fx.Paths.ToRelative(undoRecord), fx.Paths.ToRelative(undoFolder), [], [], false,
|
||||
new ConsumedEntry(fx.Paths.ToRelative(recordPath), fx.Paths.ToRelative(entryFolder), SequenceTrash.EntryLease(replaced.Name), [manifest.Frames[0].Id],
|
||||
null))
|
||||
{
|
||||
Phase = ReplacePhase.Written,
|
||||
}.Write(folder);
|
||||
var plan = await fx.Service.PlanEmptyTrashAsync(fx.Archive.Id, [replaced.Name], fx.Ct);
|
||||
Assert.Equal(replaced.Name, plan.Entries.Single().Name);
|
||||
|
||||
// The emptying settles the undo under the entry's lease before it reads the entry again, which takes the photo out of
|
||||
// it: the entry is not erased as planned, and the sequence keeps its original.
|
||||
var kept = await EmptyAsync(fx, engine, plan, JobState.Failed);
|
||||
Assert.Contains(Loc.L("Archive.Log.TrashKept", replaced.Title), kept.LogTail);
|
||||
Assert.FileMissing(folder.ReplaceJournalPath);
|
||||
Assert.Equal(undo.Name, (await fx.Service.ListTrashAsync(fx.Archive.Id, fx.Ct)).Single().Name);
|
||||
Assert.Equal(Sha(photos[0]), Sha(folder.File(Sidecar(fx, sequence).Frames[0].File)));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AnEntryNamesTheJobThatReplacedItsPhotos()
|
||||
{
|
||||
|
||||
@@ -5,6 +5,7 @@ using Chronos.Core.Localization;
|
||||
using Chronos.Core.Model;
|
||||
using Chronos.Core.Project;
|
||||
using Chronos.Core.Render;
|
||||
using Chronos.Data.Archiving;
|
||||
using Chronos.Data.Sync;
|
||||
using Chronos.Data.Tests.Archiving;
|
||||
using Chronos.Protocol;
|
||||
@@ -180,7 +181,7 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
var kept = await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct);
|
||||
Assert.True(kept.Select(k => k.Sha256).ToHashSet().SetEquals(photos[1..].Select(Sha)), "the photo deleted here stays deleted");
|
||||
Assert.True(await f.Db.FrameRemovals.IsRemovedAsync(local.Id, Sha(photos[0])), "and keeps its tombstone");
|
||||
Assert.Contains("deleted on this PC, not downloaded", done.LogTail);
|
||||
Assert.Contains(Loc.L("Archive.Log.SyncDeletedHere", remote.Frames.Single(fr => fr.Sha256 == Sha(photos[0])).FileName), done.LogTail);
|
||||
Assert.False(Directory.Exists(Path.Combine(f.Archive.Paths.StagingDirectory, "sync-" + remote.Id.ToString("N"))), "staging cleaned up");
|
||||
}
|
||||
|
||||
@@ -207,7 +208,7 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
var done = await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
Assert.Equal(3, remote.Frames.Count);
|
||||
Assert.Contains("deleted on the server, not uploaded", done.LogTail);
|
||||
Assert.Contains(Loc.L("Archive.Log.SyncDeletedThere", "DSC_0004.JPG"), done.LogTail);
|
||||
// Its hash is in the index now: the comparison leaves it out, so the row settles instead of offering Upload for good.
|
||||
Assert.True((await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct)).All(fr => fr.Sha256 is not null), "hashed by the verify");
|
||||
var entry = await f.EntryAsync(local.DisplayTitle);
|
||||
@@ -881,6 +882,36 @@ public sealed class SyncTransferTests(TestContext context)
|
||||
Assert.Equal(SyncStatus.InSync, (await f.EntryAsync("Harbour")).Status);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFrameThatTookBackItsPhotoInAnotherEncodingIsDrawnAgain()
|
||||
{
|
||||
await using var f = await SyncFixture.CreateAsync(context);
|
||||
var photos = f.NewPhotos(3);
|
||||
var local = await f.LocalAsync("Harbour", photos);
|
||||
var engine = await f.StartEngineAsync();
|
||||
var drawn = await engine.EnqueueAsync(ArchiveJobs.Thumbnails([local.Id], "Harbour"), cancellationToken: f.Ct);
|
||||
await f.Archive.WaitForAsync(drawn.Id, JobState.Done);
|
||||
var lost = (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1];
|
||||
File.Delete(f.Archive.Paths.FromRelative(lost.RelativePath));
|
||||
await f.Archive.Service.Verifier.VerifyAsync(f.Archive.Archive, [local.Id], default, (_, _, _) => ValueTask.CompletedTask, cancellationToken: f.Ct);
|
||||
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn before");
|
||||
// The server converted that photo.
|
||||
var converted = f.NewPhotos(1)[0];
|
||||
var remote = f.Remote.Add("Harbour", [photos[0], converted, photos[2]]);
|
||||
remote.Removed.Add(new RemovedPhotoDto(Sha(photos[1]), Sha(converted), "replaced", DateTimeOffset.UtcNow));
|
||||
await f.Sync.LinkAsync(f.Server, remote.Id, local.Id, f.Ct);
|
||||
await f.RefreshAsync();
|
||||
|
||||
var job = await f.Sync.EnqueueDownloadAsync(engine, f.Server, remote.Id, "Harbour", SyncDownloadMode.Originals, null);
|
||||
await f.Archive.WaitForAsync(job.Id, JobState.Done);
|
||||
|
||||
// Nothing was filed, only a frame swapped, which cleared what covered the sequence: the thumbnails it had are drawn again.
|
||||
Assert.Equal(lost.Id, (await f.Db.Frames.ListBySequenceAsync(local.Id, cancellationToken: f.Ct))[1].Id);
|
||||
var again = Assert.Contains((await f.Db.Jobs.ListAsync(null, JobKind.Thumbs, 50, null, f.Ct)).Items, j => j.Id != drawn.Id && j.TargetIds.Contains(local.Id));
|
||||
await f.Archive.WaitForAsync(again.Id, JobState.Done);
|
||||
Assert.True(((await f.Db.Sequences.GetAsync(local.Id, f.Ct))!.Status & SequenceStatus.ThumbsBuilt) != 0, "drawn again");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFrameTakesBackItsPhotoInAnotherEncodingOnlyWhereTheSequenceIsNow()
|
||||
{
|
||||
|
||||
@@ -629,6 +629,9 @@ public sealed class LibraryTests(TestContext context)
|
||||
plan = await dry.ReadAsync(ProtocolJson.Default.TrashEmptyPlanDto);
|
||||
Assert.True(plan.Entries.Single(e => e.Kind == "replaced").ConversionDeleted);
|
||||
Assert.False(plan.Entries.Single(e => e.Name == photo).ConversionDeleted);
|
||||
// The panel's confirmation names such entries first, each saying so, wherever the trash lists them.
|
||||
var page = await server.Client.GetStringAsync("/admin/pages/library/trash.js");
|
||||
Assert.Contains("list: [...plan.entries.filter((e) => e.conversionDeleted), ...plan.entries.filter((e) => !e.conversionDeleted)]", page);
|
||||
using (var empty = await admin.PostAsJsonAsync("/api/v1/archive/trash/empty", new TrashEmptyRequest(null, false, plan.Fingerprint),
|
||||
ProtocolJson.Default.TrashEmptyRequest))
|
||||
await UntilJobAsync(admin, (await empty.ReadAsync(ProtocolJson.Default.JobDto)).Id, "done");
|
||||
|
||||
Reference in new issue
Block a user