Importazione e dati: «Tieni il più grande» per gruppi, righe di _febe.jsonl mai perse, ricostruzione robusta (import-1, import-2, library-1, journal-1, rebuild-1, rebuild-2, rebuild-3, backup-1)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
This commit is contained in:
13 files changed
+1145
-95
No files matched your search
@@ -127,6 +127,7 @@ public sealed class DownloadQueue
|
||||
public async Task<DownloadReport> RunOnceAsync(CancellationToken ct)
|
||||
{
|
||||
Directory.CreateDirectory(_options.StagingFolder);
|
||||
await _ingest.AppendPendingJournalAsync(ct).ConfigureAwait(false); // lines a busy _febe.jsonl refused earlier
|
||||
var tally = new Tally();
|
||||
var attempted = new ConcurrentDictionary<long, bool>();
|
||||
var accounts = new ConcurrentDictionary<long, Account?>();
|
||||
@@ -168,6 +169,15 @@ public sealed class DownloadQueue
|
||||
}
|
||||
|
||||
private async Task ProcessAsync(PendingDownload pending, Tally tally, ConcurrentDictionary<long, Account?> accounts, ConcurrentDictionary<long, bool> singlePostQueued, CancellationToken ct)
|
||||
{
|
||||
await ProcessFileAsync(pending, tally, accounts, singlePostQueued, ct).ConfigureAwait(false);
|
||||
|
||||
// Whatever the outcome (stored, duplicate, skipped, failed), the entry may now be settled: its journal line is
|
||||
// written once no file waits any more and at least one is in the library (spec §8.5).
|
||||
await AppendJournalWhenSettledAsync(pending.Media.EntryId).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private async Task ProcessFileAsync(PendingDownload pending, Tally tally, ConcurrentDictionary<long, Account?> accounts, ConcurrentDictionary<long, bool> singlePostQueued, CancellationToken ct)
|
||||
{
|
||||
var media = pending.Media;
|
||||
if (!accounts.TryGetValue(pending.AccountId, out var account))
|
||||
@@ -319,7 +329,6 @@ public sealed class DownloadQueue
|
||||
{
|
||||
tally.Add(ref tally.Stored);
|
||||
Report(media, entry, DownloadStep.Stored);
|
||||
await AppendJournalWhenCompleteAsync(entry, ct).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -393,12 +402,17 @@ public sealed class DownloadQueue
|
||||
Report(media, entry, DownloadStep.Expired);
|
||||
}
|
||||
|
||||
private async Task AppendJournalWhenCompleteAsync(Entry entry, CancellationToken ct)
|
||||
/// <summary>
|
||||
/// Appends the journal line of an entry once settled. A journal that cannot be written now (a sync client holding
|
||||
/// <c>_febe.jsonl</c>, a share that dropped) never turns a stored file back into one to download: the line waits
|
||||
/// in <see cref="LibraryIngest.TryAppendJournalAsync"/> and is written by a later pass.
|
||||
/// </summary>
|
||||
private async Task AppendJournalWhenSettledAsync(long entryId)
|
||||
{
|
||||
var files = await _storage.Media.ListByEntryAsync(entry.Id, ct).ConfigureAwait(false);
|
||||
if (files.All(f => f.State is not (MediaState.ToDownload or MediaState.Expired)))
|
||||
var files = await _storage.Media.ListByEntryAsync(entryId, CancellationToken.None).ConfigureAwait(false);
|
||||
if (files.Any(f => EntryStateRules.IsPresent(f.State)) && files.All(f => f.State is not (MediaState.ToDownload or MediaState.Expired)))
|
||||
{
|
||||
await _ingest.AppendJournalAsync(entry.Id, ct).ConfigureAwait(false);
|
||||
await _ingest.TryAppendJournalAsync(entryId, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
var files = await LoadFilesAsync(runId, ct).ConfigureAwait(false);
|
||||
var planner = await ImportPlanner.LoadAsync(env.Storage, runId, files, ct).ConfigureAwait(false);
|
||||
var ingest = new LibraryIngest(env.Storage, env.Layout, env.Naming, env.Journal, env.Clock, env.Thumbnails, hashIndex: null, env.SetFileTimeToPublication);
|
||||
await ingest.AppendPendingJournalAsync(ct).ConfigureAwait(false); // lines a busy _febe.jsonl refused earlier
|
||||
var context = new CopyContext(run, layout, recognizers, ingest);
|
||||
var total = files.Count;
|
||||
var bytesTotal = files.Values.Sum(f => f.Bytes);
|
||||
@@ -99,6 +100,7 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
progress?.Report(new ImportProgress(runId, ImportPhase.Copying, done, total, bytesDone, bytesTotal, file.RelativePath, new Dictionary<ImportOutcome, int>(), copied, env.Clock.Now));
|
||||
}
|
||||
|
||||
await ingest.AppendPendingJournalAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
run = run with { State = ImportRunState.Completed, CompletedAt = env.Clock.Now, LastError = null };
|
||||
await env.Storage.Imports.UpdateRunAsync(run, CancellationToken.None).ConfigureAwait(false);
|
||||
await env.Storage.Events.AppendAsync(
|
||||
@@ -226,15 +228,22 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
catch (ContentChangedException)
|
||||
{
|
||||
tally.Failed++;
|
||||
return await SaveAsync(file with { State = ImportFileState.Failed, LastError = ImportText.NoteChanged }).ConfigureAwait(false);
|
||||
return await SaveAsync(await CurrentAsync(file).ConfigureAwait(false) with { State = ImportFileState.Failed, LastError = ImportText.NoteChanged }).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
tally.Failed++;
|
||||
return await SaveAsync(file with { State = ImportFileState.Failed, LastError = ImportAnalyzer.ErrorText(ex) }).ConfigureAwait(false);
|
||||
return await SaveAsync(await CurrentAsync(file).ConfigureAwait(false) with { State = ImportFileState.Failed, LastError = ImportAnalyzer.ErrorText(ex) }).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The file's record as saved during the attempt (the entry it was given, so a resumed copy reuses it instead of
|
||||
/// taking the file for one already in the library).
|
||||
/// </summary>
|
||||
private async Task<ImportFile> CurrentAsync(ImportFile file) =>
|
||||
await env.Storage.Imports.GetFileAsync(file.Id, CancellationToken.None).ConfigureAwait(false) ?? file;
|
||||
|
||||
/// <summary>Copies one file into its destination account as an imported entry (or into an entry it completes).</summary>
|
||||
private async Task<ImportFile> ImportAsync(CopyContext context, ImportFile file, Tally tally, CancellationToken ct)
|
||||
{
|
||||
@@ -267,7 +276,7 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
if (EntryStateRules.IsPresent(media.State) && string.Equals(media.Sha256, file.Sha256, StringComparison.Ordinal) && media.RelativePath is { } already)
|
||||
{
|
||||
// Stored before an interruption: only the journal line and the state are missing.
|
||||
await context.Ingest.AppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
|
||||
await context.Ingest.TryAppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
|
||||
return await FinishAsync(context, file, account, entry.Id, already, null, tally).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -329,7 +338,7 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
await env.Storage.Highlights.AddItemAsync(collection.Id, entry.Id, none).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await context.Ingest.AppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
|
||||
await context.Ingest.TryAppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
|
||||
return await FinishAsync(context, file, account, entry.Id, result.File.RelativePath, null, tally).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -444,6 +453,14 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
return await FinishAsync(context, file, (await AccountOfEntryAsync(media.EntryId, ct).ConfigureAwait(false)).Account, media.EntryId, media.RelativePath, ImportText.NoteReplaced, tally).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Alberto chose «keep the larger»: the file in the library now is what counts, not the one seen when the copy
|
||||
// was planned. A smaller or equal file never takes the place of a larger one.
|
||||
if (!ImportPlanner.IsLarger(file.Width, file.Height, file.Bytes, media.Width, media.Height, media.Bytes))
|
||||
{
|
||||
tally.Skipped++;
|
||||
return await SaveAsync(file with { State = ImportFileState.Skipped, MatchedMediaId = media.Id, LastError = ImportText.NoteLibraryLarger }).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var (account, entry) = await AccountOfEntryAsync(media.EntryId, ct).ConfigureAwait(false);
|
||||
var folder = env.Layout.AccountFolder(account);
|
||||
var source = LibraryLayout.FullPath(context.Layout.Root, file.RelativePath);
|
||||
@@ -473,9 +490,19 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
ct).ConfigureAwait(false))!;
|
||||
}
|
||||
|
||||
// From here the replacement goes to the end: a half-done replacement is worse than a late one.
|
||||
// From here the replacement goes to the end: a half-done replacement is worse than a late one. The new file is
|
||||
// either registered or removed again: never left in the library with nothing pointing to it.
|
||||
var none = CancellationToken.None;
|
||||
var onDisk = await env.FileSystem.ComputeSha256Async(saved.FullPath, none).ConfigureAwait(false);
|
||||
string? onDisk = null;
|
||||
try
|
||||
{
|
||||
onDisk = await env.FileSystem.ComputeSha256Async(saved.FullPath, none).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Not readable: not verified.
|
||||
}
|
||||
|
||||
if (!string.Equals(onDisk, file.Sha256, StringComparison.Ordinal))
|
||||
{
|
||||
TryDelete(saved.FullPath); // the new file was never registered: nothing in the library points to it
|
||||
@@ -488,14 +515,19 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
var thumbnail = false;
|
||||
if (file.MediaKind == MediaKind.Photo)
|
||||
{
|
||||
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
|
||||
await using (var stream = File.OpenRead(saved.FullPath))
|
||||
try
|
||||
{
|
||||
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
|
||||
await using var stream = File.OpenRead(saved.FullPath);
|
||||
if (ImageDecoding.ReadSize(stream) is { } size)
|
||||
{
|
||||
(width, height) = size;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Verified a moment ago: recorded without perceptual hash, with the size measured by the analysis.
|
||||
}
|
||||
|
||||
thumbnail = TryThumbnail(saved.Sha256, () => File.OpenRead(saved.FullPath));
|
||||
}
|
||||
@@ -513,7 +545,21 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
media.State,
|
||||
env.Clock.Now,
|
||||
none).ConfigureAwait(false);
|
||||
await context.Ingest.AppendJournalAsync(entry.Id, none).ConfigureAwait(false);
|
||||
await context.Ingest.TryAppendJournalAsync(entry.Id, none).ConfigureAwait(false);
|
||||
if (entry.InstagramId is null && string.IsNullOrWhiteSpace(entry.Code) && !string.Equals(oldRelative, saved.RelativePath, StringComparison.Ordinal))
|
||||
{
|
||||
// An entry without identifier or code is known in the journal by its file: the line of the old file is
|
||||
// closed («delete», not remembered), so a rebuild does not bring back an entry whose file is gone.
|
||||
try
|
||||
{
|
||||
await env.Journal.AppendDeletedAsync(folder, account, entry, [oldRelative], remember: false, none).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Without this line a rebuild reports the old file as missing and does not restore it: nothing is lost.
|
||||
}
|
||||
}
|
||||
|
||||
if (hash is { } h)
|
||||
{
|
||||
env.HashIndex?.Add(media.Id, account.Id, h);
|
||||
@@ -538,7 +584,7 @@ public sealed class ImportCopier(ImportEnvironment env)
|
||||
{
|
||||
thumbnails.Delete(oldSha);
|
||||
}
|
||||
catch (IOException)
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// A thumbnail left behind costs a few kilobytes.
|
||||
}
|
||||
|
||||
@@ -41,8 +41,10 @@ internal sealed record PlannedFile(ImportFile File, PlannedAction Action, string
|
||||
/// source;</item>
|
||||
/// <item>«Tieni quello in libreria»: the source file is skipped. «Tienili entrambi»: it is imported. «Tieni il più grande»:
|
||||
/// if the library file is larger the source file is skipped, otherwise it replaces the library file (more pixels win,
|
||||
/// then more bytes; a tie keeps the library file). Inside the source, among files joined by «tieni il più grande» only
|
||||
/// the largest is imported;</item>
|
||||
/// then more bytes; a tie keeps the library file). Several source files paired with the same library file form one
|
||||
/// group: only the largest of them can replace it, the others are skipped, so a replacement never removes the copy
|
||||
/// another replacement has just put in the library. Inside the source, among files joined by «tieni il più grande»
|
||||
/// only the largest is imported;</item>
|
||||
/// <item>files of a folder without account stay out.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
@@ -88,6 +90,7 @@ internal sealed class ImportPlanner
|
||||
}
|
||||
|
||||
var groups = new UnionFind();
|
||||
var rivals = new Dictionary<long, List<long>>(); // library file → source files that may replace it
|
||||
foreach (var pair in pairs.Where(p => p.Decision != DuplicateDecision.Pending && p.CandidateImportFileId is not null))
|
||||
{
|
||||
var candidate = pair.CandidateImportFileId!.Value;
|
||||
@@ -99,17 +102,13 @@ internal sealed class ImportPlanner
|
||||
planner.Lose(candidate, ImportText.NoteKeptExisting);
|
||||
break;
|
||||
case DuplicateDecision.KeepLarger:
|
||||
var library = await storage.Media.GetAsync(mediaId, ct).ConfigureAwait(false);
|
||||
if (library is not null && files.TryGetValue(candidate, out var file)
|
||||
&& IsLarger(file.Width, file.Height, file.Bytes, library.Width, library.Height, library.Bytes))
|
||||
if (!rivals.TryGetValue(mediaId, out var list))
|
||||
{
|
||||
planner._replacements[candidate] = mediaId;
|
||||
}
|
||||
else
|
||||
{
|
||||
planner.Lose(candidate, ImportText.NoteLibraryLarger);
|
||||
list = [];
|
||||
rivals[mediaId] = list;
|
||||
}
|
||||
|
||||
list.Add(candidate);
|
||||
break;
|
||||
default:
|
||||
break; // KeepBoth: imported
|
||||
@@ -134,20 +133,54 @@ internal sealed class ImportPlanner
|
||||
foreach (var group in groups.Groups())
|
||||
{
|
||||
var members = group.Where(files.ContainsKey).Select(id => files[id]).ToList();
|
||||
var winner = members
|
||||
.OrderByDescending(f => (long)(f.Width ?? 0) * (f.Height ?? 0))
|
||||
.ThenByDescending(f => f.Bytes)
|
||||
.ThenBy(f => f.Id)
|
||||
.FirstOrDefault();
|
||||
var winner = Largest(members);
|
||||
foreach (var member in members.Where(m => m.Id != winner?.Id))
|
||||
{
|
||||
planner.Lose(member.Id, ImportText.NoteKeptOtherInSource);
|
||||
}
|
||||
}
|
||||
|
||||
// «Tieni il più grande» against a library file: the library file and every source file paired with it are one
|
||||
// group, and only the largest of the group stays. Deciding pair by pair would let a second, smaller source
|
||||
// file replace the copy the first one has just put in the library (and, with «Sposta», lose the larger one).
|
||||
foreach (var (mediaId, candidates) in rivals)
|
||||
{
|
||||
var library = await storage.Media.GetAsync(mediaId, ct).ConfigureAwait(false);
|
||||
// A file already skipped never entered the library: it does not compete.
|
||||
var members = candidates.Distinct()
|
||||
.Where(id => files.TryGetValue(id, out var f) && f.State != ImportFileState.Skipped && !planner._decided.ContainsKey(id))
|
||||
.Select(id => files[id])
|
||||
.ToList();
|
||||
var winner = Largest(members);
|
||||
var replaces = winner is not null && library is not null
|
||||
&& (IsLarger(winner.Width, winner.Height, winner.Bytes, library.Width, library.Height, library.Bytes)
|
||||
// The library already holds this very file: a replacement interrupted before its state was saved,
|
||||
// which the copy finishes.
|
||||
|| string.Equals(winner.Sha256, library.Sha256, StringComparison.Ordinal));
|
||||
foreach (var member in members)
|
||||
{
|
||||
if (replaces && member.Id == winner!.Id)
|
||||
{
|
||||
planner._replacements[member.Id] = mediaId;
|
||||
}
|
||||
else
|
||||
{
|
||||
planner.Lose(member.Id, replaces ? ImportText.NoteLargerInSource : ImportText.NoteLibraryLarger);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return planner;
|
||||
}
|
||||
|
||||
/// <summary>The largest file: more pixels, then more bytes; on a tie the first one analysed.</summary>
|
||||
private static ImportFile? Largest(IEnumerable<ImportFile> files) =>
|
||||
files
|
||||
.OrderByDescending(f => (long)(f.Width ?? 0) * (f.Height ?? 0))
|
||||
.ThenByDescending(f => f.Bytes)
|
||||
.ThenBy(f => f.Id)
|
||||
.FirstOrDefault();
|
||||
|
||||
/// <summary>True when the folder of the file has (or will have) an account.</summary>
|
||||
public bool HasTarget(ImportFile file) => file.TargetAccountId is not null || _newAccounts.ContainsKey(file.Folder);
|
||||
|
||||
|
||||
@@ -136,6 +136,7 @@ public static class ImportText
|
||||
internal const string NoteKeptExisting = "Hai scelto di tenere il file già in libreria.";
|
||||
internal const string NoteLibraryLarger = "Hai scelto di tenere il più grande: è quello già in libreria.";
|
||||
internal const string NoteKeptOtherInSource = "Hai scelto di tenere l'altro file della sorgente.";
|
||||
internal const string NoteLargerInSource = "Hai scelto di tenere il più grande: è un altro file della sorgente, che prende il posto di quello in libreria.";
|
||||
internal const string NoteNotRecognized = "Né foto né video: ignorato.";
|
||||
internal const string NoteAlreadyInLibrary = "Identico a un file già in libreria: saltato.";
|
||||
internal const string NoteReplaced = "Ha sostituito il file più piccolo della libreria, come hai scelto.";
|
||||
|
||||
@@ -25,6 +25,9 @@ public static class Mp4Reader
|
||||
|
||||
private const int MaxTopLevelBoxes = 10_000;
|
||||
|
||||
/// <summary>The most child boxes read inside one box (real files have a few dozen).</summary>
|
||||
private const int MaxChildBoxes = 10_000;
|
||||
|
||||
/// <summary>Reads a file (read-only). Null when no <c>moov</c> box is found.</summary>
|
||||
/// <param name="path">The file.</param>
|
||||
/// <param name="now">Current time: creation times after it (by more than a day) are ignored.</param>
|
||||
@@ -72,7 +75,8 @@ public static class Mp4Reader
|
||||
size = length - position; // to the end of the file
|
||||
}
|
||||
|
||||
if (size < headerSize || position + size > length + 8)
|
||||
// Compared without adding to the position: a 64-bit size close to 2^63 would overflow the sum and pass.
|
||||
if (size < headerSize || size > length - position + 8)
|
||||
{
|
||||
return null; // damaged
|
||||
}
|
||||
@@ -193,7 +197,7 @@ public static class Mp4Reader
|
||||
{
|
||||
var list = new List<(string, int, int)>();
|
||||
var offset = 0;
|
||||
while (offset + 8 <= data.Length)
|
||||
while (offset + 8 <= data.Length && list.Count < MaxChildBoxes)
|
||||
{
|
||||
long size = BinaryPrimitives.ReadUInt32BigEndian(data.Slice(offset, 4));
|
||||
var type = System.Text.Encoding.Latin1.GetString(data.Slice(offset + 4, 4));
|
||||
@@ -208,7 +212,9 @@ public static class Mp4Reader
|
||||
size = data.Length - offset;
|
||||
}
|
||||
|
||||
if (size < headerSize || offset + size > data.Length)
|
||||
// «size > remaining» cannot overflow (unlike «offset + size»), and with «size >= headerSize» it keeps the
|
||||
// offset moving forward: a damaged 64-bit size can neither loop back nor run past the end.
|
||||
if (size < headerSize || size > data.Length - offset)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Febe.Core.Abstractions;
|
||||
@@ -18,6 +19,11 @@ public sealed record BackupResult(BackupFile? Created, IReadOnlyList<string> Rem
|
||||
/// Daily copy of the database (spec §8.5): <c>VACUUM INTO</c> a new file <c>Dati/copie/febe-YYYY-MM-DD.sqlite</c>
|
||||
/// (local date), keeping seven days. A copy is written under a temporary name and renamed, so a cut copy never has a
|
||||
/// final name. Only files named <c>febe-YYYY-MM-DD.sqlite</c> are ever removed.
|
||||
/// <para>
|
||||
/// The 3:20 loop and «Fai una copia adesso» each make their own instance: copies and retention of one folder take turns
|
||||
/// (one gate per folder), and the retention removes only temporary copies left behind long ago, never one still being
|
||||
/// written.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock clock, string backupsFolder, int keepDays = DailyBackup.DefaultKeepDays)
|
||||
{
|
||||
@@ -27,6 +33,12 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
|
||||
private const string Extension = ".sqlite";
|
||||
private const string PartSuffix = ".part";
|
||||
|
||||
/// <summary>A temporary copy untouched for this long was left by a copy that stopped (crash, power cut).</summary>
|
||||
private static readonly TimeSpan StalePart = TimeSpan.FromHours(6);
|
||||
|
||||
private static readonly ConcurrentDictionary<string, SemaphoreSlim> Gates = new(
|
||||
OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal);
|
||||
|
||||
[GeneratedRegex(@"^febe-(\d{4}-\d{2}-\d{2})\.sqlite$", RegexOptions.CultureInvariant)]
|
||||
private static partial Regex BackupName();
|
||||
|
||||
@@ -36,32 +48,39 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
|
||||
/// <summary>File name of the copy of a day.</summary>
|
||||
public static string FileNameFor(DateOnly day) => Prefix + day.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) + Extension;
|
||||
|
||||
private SemaphoreSlim Gate => Gates.GetOrAdd(Folder, _ => new SemaphoreSlim(1, 1));
|
||||
|
||||
/// <summary>Makes today's copy if missing, then applies the retention. Safe to call often.</summary>
|
||||
public async Task<BackupResult> RunAsync(CancellationToken ct = default)
|
||||
{
|
||||
Directory.CreateDirectory(Folder);
|
||||
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
|
||||
var target = Path.Combine(Folder, FileNameFor(today));
|
||||
BackupFile? created = null;
|
||||
if (!File.Exists(target))
|
||||
await Gate.WaitAsync(ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
|
||||
try
|
||||
Directory.CreateDirectory(Folder);
|
||||
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
|
||||
var target = Path.Combine(Folder, FileNameFor(today));
|
||||
BackupFile? created = null;
|
||||
if (!File.Exists(target))
|
||||
{
|
||||
await database.BackupToAsync(part, ct).ConfigureAwait(false);
|
||||
File.Move(part, target, overwrite: false);
|
||||
created = new BackupFile(today, target, new FileInfo(target).Length);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(part))
|
||||
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
|
||||
try
|
||||
{
|
||||
File.Delete(part);
|
||||
await database.BackupToAsync(part, ct).ConfigureAwait(false);
|
||||
File.Move(part, target, overwrite: false);
|
||||
created = new BackupFile(today, target, new FileInfo(target).Length);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(part);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new BackupResult(created, ApplyRetention(today));
|
||||
return new BackupResult(created, Retain(today));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -71,26 +90,31 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
|
||||
/// </summary>
|
||||
public async Task<BackupResult> RunNowAsync(CancellationToken ct = default)
|
||||
{
|
||||
Directory.CreateDirectory(Folder);
|
||||
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
|
||||
var target = Path.Combine(Folder, FileNameFor(today));
|
||||
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
|
||||
BackupFile created;
|
||||
await Gate.WaitAsync(ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await database.BackupToAsync(part, ct).ConfigureAwait(false);
|
||||
File.Move(part, target, overwrite: true);
|
||||
created = new BackupFile(today, target, new FileInfo(target).Length);
|
||||
Directory.CreateDirectory(Folder);
|
||||
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
|
||||
var target = Path.Combine(Folder, FileNameFor(today));
|
||||
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
|
||||
BackupFile created;
|
||||
try
|
||||
{
|
||||
await database.BackupToAsync(part, ct).ConfigureAwait(false);
|
||||
File.Move(part, target, overwrite: true);
|
||||
created = new BackupFile(today, target, new FileInfo(target).Length);
|
||||
}
|
||||
finally
|
||||
{
|
||||
TryDelete(part);
|
||||
}
|
||||
|
||||
return new BackupResult(created, Retain(today));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(part))
|
||||
{
|
||||
File.Delete(part);
|
||||
}
|
||||
Gate.Release();
|
||||
}
|
||||
|
||||
return new BackupResult(created, ApplyRetention(today));
|
||||
}
|
||||
|
||||
/// <summary>The copies present, newest first.</summary>
|
||||
@@ -114,10 +138,25 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes copies older than <see cref="DefaultKeepDays"/> days (today included) and stale temporary copies.
|
||||
/// Returns the removed paths.
|
||||
/// Removes copies older than <see cref="DefaultKeepDays"/> days (today included) and temporary copies left behind
|
||||
/// by a copy that stopped (untouched for hours: one still being written is never removed). A file that cannot be
|
||||
/// removed now (open in another program) is left for the next time. Returns the removed paths.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> ApplyRetention(DateOnly today)
|
||||
{
|
||||
Gate.Wait();
|
||||
try
|
||||
{
|
||||
return Retain(today);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Gate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The retention; the caller holds the gate.</summary>
|
||||
private List<string> Retain(DateOnly today)
|
||||
{
|
||||
var removed = new List<string>();
|
||||
if (!Directory.Exists(Folder))
|
||||
@@ -128,19 +167,42 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
|
||||
var oldestKept = today.AddDays(-(Math.Max(1, keepDays) - 1));
|
||||
foreach (var backup in List().Where(b => b.Day < oldestKept))
|
||||
{
|
||||
File.Delete(backup.Path);
|
||||
removed.Add(backup.Path);
|
||||
if (TryDelete(backup.Path))
|
||||
{
|
||||
removed.Add(backup.Path);
|
||||
}
|
||||
}
|
||||
|
||||
var staleBefore = clock.Now.UtcDateTime - StalePart;
|
||||
foreach (var part in Directory.EnumerateFiles(Folder, Prefix + "*" + PartSuffix))
|
||||
{
|
||||
File.Delete(part);
|
||||
removed.Add(part);
|
||||
if (File.GetLastWriteTimeUtc(part) < staleBefore && TryDelete(part))
|
||||
{
|
||||
removed.Add(part);
|
||||
}
|
||||
}
|
||||
|
||||
return removed;
|
||||
}
|
||||
|
||||
private static bool TryDelete(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
File.Delete(path);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
return false; // the next retention tries again
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The day of a backup file name, or null for any other file.</summary>
|
||||
public static DateOnly? TryParseDay(string fileName)
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Globalization;
|
||||
using Febe.Core.Abstractions;
|
||||
using Febe.Core.Domain;
|
||||
using Febe.Core.Storage;
|
||||
@@ -87,6 +88,14 @@ public sealed class LibraryIngest(
|
||||
bool setFileTimeToPublication = true,
|
||||
SafeFileWriter? writer = null)
|
||||
{
|
||||
/// <summary>
|
||||
/// Setting with the entries whose <c>_febe.jsonl</c> line is still to write (ids, comma separated): it survives a
|
||||
/// restart, and downloads and imports share it.
|
||||
/// </summary>
|
||||
internal const string PendingJournalKey = "libreria.giornale-da-scrivere";
|
||||
|
||||
private static readonly SemaphoreSlim PendingJournalGate = new(1, 1);
|
||||
|
||||
private readonly SafeFileWriter _writer = writer ?? new SafeFileWriter(layout.MaxPathLength);
|
||||
private volatile LibraryNaming _naming = naming;
|
||||
private volatile bool _setFileTimeToPublication = setFileTimeToPublication;
|
||||
@@ -191,19 +200,27 @@ public sealed class LibraryIngest(
|
||||
return new MediaSaveResult(MediaSaveOutcome.IdenticalInAccount, null, identical, []);
|
||||
}
|
||||
|
||||
// From here the file is in the library under its final name: it is always recorded, even when reading it again
|
||||
// for the perceptual hash and the size fails (callers treat an IOException as «nothing written»).
|
||||
ulong? hash = null;
|
||||
int? width = media.Width, height = media.Height;
|
||||
var hasThumbnail = false;
|
||||
if (media.Kind == MediaKind.Photo)
|
||||
{
|
||||
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
|
||||
await using (var stream = File.OpenRead(saved.FullPath))
|
||||
try
|
||||
{
|
||||
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
|
||||
await using var stream = File.OpenRead(saved.FullPath);
|
||||
if (ImageDecoding.ReadSize(stream) is { } size)
|
||||
{
|
||||
(width, height) = size;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Recorded without perceptual hash (no duplicate check by look for this file) and with the size the
|
||||
// page gave; a rebuild of the database computes both again.
|
||||
}
|
||||
|
||||
hasThumbnail = TryThumbnail(saved.Sha256, () => File.OpenRead(saved.FullPath), owned: true);
|
||||
}
|
||||
@@ -261,6 +278,129 @@ public sealed class LibraryIngest(
|
||||
ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Appends the <c>_febe.jsonl</c> line of an entry whose files are already in the library. When the journal cannot
|
||||
/// be written now (held by a sync client, a network share that dropped, a full disk), the files stay recorded as
|
||||
/// they are and the entry is remembered: <see cref="AppendPendingJournalAsync"/> writes its line later, so a rebuild
|
||||
/// of the database still finds it (spec §8.5). Returns false in that case.
|
||||
/// </summary>
|
||||
public async Task<bool> TryAppendJournalAsync(long entryId, CancellationToken ct = default)
|
||||
{
|
||||
try
|
||||
{
|
||||
await AppendJournalAsync(entryId, ct).ConfigureAwait(false);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
await RememberPendingJournalAsync(entryId).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the journal lines that could not be written before (see <see cref="TryAppendJournalAsync"/>), each with the
|
||||
/// entry as it is now. Entries deleted meanwhile, or left without files, need no line. Returns how many still wait.
|
||||
/// </summary>
|
||||
public async Task<int> AppendPendingJournalAsync(CancellationToken ct = default)
|
||||
{
|
||||
await PendingJournalGate.WaitAsync(ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var pending = await ReadPendingJournalAsync().ConfigureAwait(false);
|
||||
if (pending.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
var left = new List<long>();
|
||||
foreach (var entryId in pending)
|
||||
{
|
||||
var stored = await storage.Entries.GetWithMediaAsync(entryId, CancellationToken.None).ConfigureAwait(false);
|
||||
if (stored is null || stored.Entry.State == EntryState.Deleted || !stored.Media.Any(m => EntryStateRules.IsPresent(m.State)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
left.Add(entryId);
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await AppendJournalAsync(entryId, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
left.Add(entryId);
|
||||
}
|
||||
}
|
||||
|
||||
await WritePendingJournalAsync(left).ConfigureAwait(false);
|
||||
return left.Count;
|
||||
}
|
||||
finally
|
||||
{
|
||||
PendingJournalGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RememberPendingJournalAsync(long entryId)
|
||||
{
|
||||
await PendingJournalGate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
var pending = await ReadPendingJournalAsync().ConfigureAwait(false);
|
||||
if (pending.Contains(entryId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await WritePendingJournalAsync([.. pending, entryId]).ConfigureAwait(false);
|
||||
if (pending.Count == 0)
|
||||
{
|
||||
var accountId = (await storage.Entries.GetAsync(entryId, CancellationToken.None).ConfigureAwait(false))?.AccountId;
|
||||
await storage.Events.AppendAsync(
|
||||
new EventRecord
|
||||
{
|
||||
At = clock.Now,
|
||||
Code = "giornale-in-sospeso",
|
||||
Severity = EventSeverity.Warning,
|
||||
Message = "Non si è potuto scrivere _febe.jsonl nella cartella dell'account (file in uso da un altro programma o disco non raggiungibile). I file sono al sicuro in libreria: Febe scrive le righe mancanti appena può.",
|
||||
AccountId = accountId,
|
||||
},
|
||||
CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
PendingJournalGate.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<List<long>> ReadPendingJournalAsync()
|
||||
{
|
||||
var text = await storage.Settings.GetAsync(PendingJournalKey, CancellationToken.None).ConfigureAwait(false);
|
||||
var ids = new List<long>();
|
||||
foreach (var part in (text ?? string.Empty).Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
|
||||
{
|
||||
if (long.TryParse(part, NumberStyles.Integer, CultureInfo.InvariantCulture, out var id) && !ids.Contains(id))
|
||||
{
|
||||
ids.Add(id);
|
||||
}
|
||||
}
|
||||
|
||||
return ids;
|
||||
}
|
||||
|
||||
private Task WritePendingJournalAsync(IReadOnlyList<long> ids) =>
|
||||
storage.Settings.SetAsync(
|
||||
PendingJournalKey,
|
||||
ids.Count == 0 ? null : string.Join(',', ids.Select(id => id.ToString(CultureInfo.InvariantCulture))),
|
||||
CancellationToken.None);
|
||||
|
||||
private bool TryThumbnail(string key, Func<Stream> open, bool owned)
|
||||
{
|
||||
if (thumbnails is null)
|
||||
|
||||
@@ -48,6 +48,12 @@ public sealed record RebuildReport
|
||||
|
||||
public int FilesMissing { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Files that are on disk but could not be read (locked by another program, no permission, damaged disk): kept as
|
||||
/// the journal describes them, without verification, and listed in <see cref="Notes"/>.
|
||||
/// </summary>
|
||||
public int FilesUnreadable { get; init; }
|
||||
|
||||
/// <summary>Files whose SHA-256 differs from the journal (the file on disk wins).</summary>
|
||||
public int HashMismatches { get; init; }
|
||||
|
||||
@@ -113,7 +119,7 @@ public sealed class LibraryRebuilder(
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
await RebuildFolderAsync(folders[i], options, report, ct).ConfigureAwait(false);
|
||||
progress?.Report(new RebuildProgress(folders[i], i + 1, folders.Count, report.FilesVerified + report.FilesMissing));
|
||||
progress?.Report(new RebuildProgress(folders[i], i + 1, folders.Count, report.FilesVerified + report.FilesMissing + report.FilesUnreadable));
|
||||
}
|
||||
|
||||
report.AccountFolders = folders.Count;
|
||||
@@ -122,7 +128,18 @@ public sealed class LibraryRebuilder(
|
||||
|
||||
private async Task RebuildFolderAsync(string folder, RebuildOptions options, Accumulator report, CancellationToken ct)
|
||||
{
|
||||
var journal = await AccountJournal.ReadAsync(folder, ct).ConfigureAwait(false);
|
||||
JournalContent journal;
|
||||
try
|
||||
{
|
||||
journal = await AccountJournal.ReadAsync(folder, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// One unreadable journal does not stop the others: this folder can be rebuilt again later.
|
||||
report.Note($"«{Path.GetFileName(folder)}»: _febe.jsonl non si può leggere ({Reason(ex)}); la cartella non è stata ricostruita.");
|
||||
return;
|
||||
}
|
||||
|
||||
report.InvalidLines += journal.InvalidLines;
|
||||
if (journal.Lines.Count == 0)
|
||||
{
|
||||
@@ -157,8 +174,26 @@ public sealed class LibraryRebuilder(
|
||||
foreach (var item in items)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (!item.Deleted && !item.HasIdentity && item.Files.Count > 0 && item.Files.Keys.All(n => checks[(item, n)].IsMissing))
|
||||
{
|
||||
// An entry without identifier or code whose files are all gone has nothing to show and nothing to download
|
||||
// again: it is reported, not restored. (A journal written before file replacements recorded the old path
|
||||
// as deleted holds such an entry for every imported file replaced by «Tieni il più grande».)
|
||||
foreach (var file in item.Files.Values)
|
||||
{
|
||||
report.FilesMissing++;
|
||||
report.Note($"File mancante: {file.Path} (elemento senza identificativo, non ripristinato)");
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
var entry = BuildEntry(account.Id, item);
|
||||
var media = item.Files.Values.OrderBy(f => f.N).Select(f => BuildMedia(item, f, item.Deleted ? null : checks[(item, f.N)], report)).ToList();
|
||||
var media = item.Files.Values
|
||||
.Select(f => BuildMedia(item, f, item.Deleted ? null : checks[(item, f.N)], report))
|
||||
.Concat(item.Removed.Values.Select(BuildRemoved))
|
||||
.OrderBy(m => m.Position)
|
||||
.ToList();
|
||||
batch.Add((item, new NewEntry(entry, media)));
|
||||
if (batch.Count >= BatchSize)
|
||||
{
|
||||
@@ -192,7 +227,8 @@ public sealed class LibraryRebuilder(
|
||||
{
|
||||
var present = item.InstagramId is { } id
|
||||
? known.Contains((item.Kind, id))
|
||||
: item.Files.Values.Any(f => paths.ContainsKey(LibraryLayout.ToStored(f.Path)));
|
||||
: item.Files.Values.Any(f => paths.ContainsKey(LibraryLayout.ToStored(f.Path)))
|
||||
|| (item.Deleted && await IsDeletionKnownAsync(account, item, ct).ConfigureAwait(false));
|
||||
if (!present)
|
||||
{
|
||||
missing.Add(item);
|
||||
@@ -216,6 +252,24 @@ public sealed class LibraryRebuilder(
|
||||
return missing;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A remembered deletion of an entry without identifier has no file path left in the database: it is recognised by
|
||||
/// the SHA-256 of its files, so a second rebuild does not add it again.
|
||||
/// </summary>
|
||||
private async Task<bool> IsDeletionKnownAsync(Account account, Item item, CancellationToken ct)
|
||||
{
|
||||
foreach (var sha in item.Files.Values.Select(f => f.Sha256).OfType<string>().Where(s => s.Length == 64))
|
||||
{
|
||||
var same = await storage.Media.FindBySha256Async(sha.ToLowerInvariant(), ct).ConfigureAwait(false);
|
||||
if (same.Any(s => s.Entry.AccountId == account.Id && s.Entry.Kind == item.Kind && s.Entry.State == EntryState.Deleted))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private async Task<Account> ResolveAccountAsync(string folder, IReadOnlyList<JournalLine> lines, Accumulator report, CancellationToken ct)
|
||||
{
|
||||
var last = lines[^1].Account;
|
||||
@@ -255,7 +309,11 @@ public sealed class LibraryRebuilder(
|
||||
return await storage.Accounts.AddAsync(created, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Replays the journal: the last save of an entry wins, deletions remove it or mark it.</summary>
|
||||
/// <summary>
|
||||
/// Replays the journal: the last save of an entry wins, deletions remove it or mark it. Every save line lists all
|
||||
/// the files present at that moment, so a position a later save no longer lists was removed (one file of a series
|
||||
/// dropped by a duplicate decision): it comes back as «eliminato», never as a file to find or download again.
|
||||
/// </summary>
|
||||
private static List<Item> Fold(IReadOnlyList<JournalLine> lines, Accumulator report)
|
||||
{
|
||||
var order = new List<Item>();
|
||||
@@ -286,11 +344,20 @@ public sealed class LibraryRebuilder(
|
||||
else
|
||||
{
|
||||
item.Last = line;
|
||||
if (line.Files is { Count: > 0 } listed)
|
||||
{
|
||||
foreach (var dropped in item.Files.Keys.Where(n => !listed.Any(f => f.N == n)).ToList())
|
||||
{
|
||||
item.Removed[dropped] = item.Files[dropped];
|
||||
item.Files.Remove(dropped);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var file in line.Files ?? [])
|
||||
{
|
||||
item.Files[file.N] = file;
|
||||
item.Removed.Remove(file.N);
|
||||
}
|
||||
}
|
||||
else if (line.Remember == true)
|
||||
@@ -315,6 +382,10 @@ public sealed class LibraryRebuilder(
|
||||
return order;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The identity of an entry across its lines: the Instagram id; else the code (an imported entry known by its code
|
||||
/// keeps it when one of its files is replaced); else the first file, for entries without either.
|
||||
/// </summary>
|
||||
private static string KeyOf(EntryKind kind, JournalLine line)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(line.Entry.Id))
|
||||
@@ -322,6 +393,11 @@ public sealed class LibraryRebuilder(
|
||||
return $"{kind}|id|{line.Entry.Id.Trim()}";
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(line.Entry.Code))
|
||||
{
|
||||
return $"{kind}|code|{line.Entry.Code.Trim()}";
|
||||
}
|
||||
|
||||
var firstPath = line.Files?.OrderBy(f => f.N).FirstOrDefault()?.Path;
|
||||
return firstPath is not null
|
||||
? $"{kind}|path|{firstPath}"
|
||||
@@ -345,47 +421,71 @@ public sealed class LibraryRebuilder(
|
||||
return FileCheck.Missing;
|
||||
}
|
||||
|
||||
// A file that cannot be read (locked by another program, no permission, bad sectors, a cloud placeholder that
|
||||
// cannot be fetched offline) must not stop the rebuild of everything else.
|
||||
string sha;
|
||||
long bytes;
|
||||
await using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 128 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
try
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 128 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
bytes = stream.Length;
|
||||
sha = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, ct).ConfigureAwait(false));
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
var recorded = file.Sha256 is { Length: 64 } s ? s.ToLowerInvariant() : null;
|
||||
return FileCheck.CannotRead(Reason(ex)) with { HasThumbnail = recorded is not null && (thumbnails?.Exists(recorded) ?? false) };
|
||||
}
|
||||
|
||||
var isVideo = string.Equals(file.Type, "video", StringComparison.OrdinalIgnoreCase);
|
||||
ulong? phash = null;
|
||||
int? width = file.Width, height = file.Height;
|
||||
var thumbnail = thumbnails?.Exists(sha) ?? false;
|
||||
if (!isVideo)
|
||||
try
|
||||
{
|
||||
phash = PerceptualHash.ComputeFromFile(path);
|
||||
await using (var stream = File.OpenRead(path))
|
||||
if (!isVideo)
|
||||
{
|
||||
if (ImageDecoding.ReadSize(stream) is { } size)
|
||||
phash = PerceptualHash.ComputeFromFile(path);
|
||||
await using (var stream = File.OpenRead(path))
|
||||
{
|
||||
(width, height) = size;
|
||||
if (ImageDecoding.ReadSize(stream) is { } size)
|
||||
{
|
||||
(width, height) = size;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Hashed but not decoded a second time: the file stays, without perceptual hash.
|
||||
}
|
||||
|
||||
if (createThumbnails && thumbnails is not null && !thumbnail)
|
||||
try
|
||||
{
|
||||
if (!isVideo && createThumbnails && thumbnails is not null && !thumbnail)
|
||||
{
|
||||
thumbnail = thumbnails.CreateFromFile(sha, path);
|
||||
}
|
||||
}
|
||||
else if (createThumbnails && thumbnails is not null && !thumbnail && frames is not null)
|
||||
{
|
||||
var frame = await frames.ExtractJpegAsync(path, ct).ConfigureAwait(false);
|
||||
if (frame is not null)
|
||||
else if (isVideo && createThumbnails && thumbnails is not null && !thumbnail && frames is not null)
|
||||
{
|
||||
using var stream = new MemoryStream(frame);
|
||||
thumbnail = thumbnails.Create(sha, stream);
|
||||
var frame = await frames.ExtractJpegAsync(path, ct).ConfigureAwait(false);
|
||||
if (frame is not null)
|
||||
{
|
||||
using var stream = new MemoryStream(frame);
|
||||
thumbnail = thumbnails.Create(sha, stream);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
thumbnail = false; // the neutral tile until a thumbnail can be made
|
||||
}
|
||||
|
||||
return new FileCheck(true, bytes, sha, phash, width, height, thumbnail);
|
||||
}
|
||||
|
||||
private static string Reason(Exception ex) => ex is UnauthorizedAccessException ? "permesso negato" : ex.Message;
|
||||
|
||||
private static Entry BuildEntry(long accountId, Item item)
|
||||
{
|
||||
var e = item.Last.Entry;
|
||||
@@ -429,6 +529,23 @@ public sealed class LibraryRebuilder(
|
||||
return media with { State = MediaState.Deleted };
|
||||
}
|
||||
|
||||
var imported = string.Equals(item.Last.Entry.Provenance, "imported", StringComparison.OrdinalIgnoreCase);
|
||||
if (check is { Unreadable: { } reason })
|
||||
{
|
||||
// The file is where the journal says: it stays linked as recorded there (nothing to download again), and
|
||||
// the report says it could not be verified.
|
||||
report.FilesUnreadable++;
|
||||
report.Note($"File illeggibile, ripristinato senza verifica: {file.Path} ({reason})");
|
||||
return media with
|
||||
{
|
||||
State = imported ? MediaState.Imported : MediaState.Downloaded,
|
||||
RelativePath = LibraryLayout.ToStored(file.Path),
|
||||
HasThumbnail = check.HasThumbnail,
|
||||
DownloadedAt = item.Last.At,
|
||||
LastError = $"File illeggibile durante la ricostruzione del database: {reason}",
|
||||
};
|
||||
}
|
||||
|
||||
if (check is not { Exists: true })
|
||||
{
|
||||
report.FilesMissing++;
|
||||
@@ -443,7 +560,6 @@ public sealed class LibraryRebuilder(
|
||||
report.Note($"Contenuto diverso da quello registrato: {file.Path}");
|
||||
}
|
||||
|
||||
var imported = string.Equals(item.Last.Entry.Provenance, "imported", StringComparison.OrdinalIgnoreCase);
|
||||
return media with
|
||||
{
|
||||
State = imported ? MediaState.Imported : MediaState.Downloaded,
|
||||
@@ -523,10 +639,24 @@ public sealed class LibraryRebuilder(
|
||||
batch.Clear();
|
||||
}
|
||||
|
||||
/// <summary>A file of a series removed by Alberto after it was saved: known, settled, never downloaded again.</summary>
|
||||
private static Media BuildRemoved(JournalFile file) => new()
|
||||
{
|
||||
Position = file.N,
|
||||
Kind = string.Equals(file.Type, "video", StringComparison.OrdinalIgnoreCase) ? MediaKind.Video : MediaKind.Photo,
|
||||
InstagramId = file.Id,
|
||||
Width = file.Width,
|
||||
Height = file.Height,
|
||||
Duration = file.DurationMs is { } ms ? TimeSpan.FromMilliseconds(ms) : null,
|
||||
Sha256 = file.Sha256 is { Length: 64 } s ? s.ToLowerInvariant() : null,
|
||||
Bytes = file.Bytes,
|
||||
State = MediaState.Deleted,
|
||||
};
|
||||
|
||||
private static void CountUntracked(string folder, List<Item> items, Accumulator report)
|
||||
{
|
||||
var known = new HashSet<string>(
|
||||
items.SelectMany(i => i.Files.Values).Select(f => LibraryLayout.ToStored(f.Path)),
|
||||
items.SelectMany(i => i.Files.Values.Concat(i.Removed.Values)).Select(f => LibraryLayout.ToStored(f.Path)),
|
||||
OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal);
|
||||
foreach (var file in Directory.EnumerateFiles(folder, "*", new EnumerationOptions { RecurseSubdirectories = true, IgnoreInaccessible = true }))
|
||||
{
|
||||
@@ -548,6 +678,13 @@ public sealed class LibraryRebuilder(
|
||||
private sealed record FileCheck(bool Exists, long Bytes, string Sha256, ulong? PerceptualHash, int? Width, int? Height, bool HasThumbnail)
|
||||
{
|
||||
public static FileCheck Missing { get; } = new(false, 0, string.Empty, null, null, null, false);
|
||||
|
||||
/// <summary>Set when the file is on disk but could not be read: why.</summary>
|
||||
public string? Unreadable { get; init; }
|
||||
|
||||
public bool IsMissing => !Exists && Unreadable is null;
|
||||
|
||||
public static FileCheck CannotRead(string reason) => Missing with { Unreadable = reason };
|
||||
}
|
||||
|
||||
private sealed class Item(EntryKind kind, JournalLine first)
|
||||
@@ -560,8 +697,14 @@ public sealed class LibraryRebuilder(
|
||||
|
||||
public string? InstagramId => string.IsNullOrWhiteSpace(Last.Entry.Id) ? null : Last.Entry.Id.Trim();
|
||||
|
||||
/// <summary>An Instagram id or a code: the entry is more than its files.</summary>
|
||||
public bool HasIdentity => InstagramId is not null || !string.IsNullOrWhiteSpace(Last.Entry.Code);
|
||||
|
||||
public SortedDictionary<int, JournalFile> Files { get; } = [];
|
||||
|
||||
/// <summary>Positions listed by an earlier save line and no longer by the last one: removed by Alberto.</summary>
|
||||
public SortedDictionary<int, JournalFile> Removed { get; } = [];
|
||||
|
||||
public bool Deleted { get; set; }
|
||||
|
||||
public DateTimeOffset? DeletedAt { get; set; }
|
||||
@@ -589,6 +732,8 @@ public sealed class LibraryRebuilder(
|
||||
|
||||
public int FilesMissing { get; set; }
|
||||
|
||||
public int FilesUnreadable { get; set; }
|
||||
|
||||
public int HashMismatches { get; set; }
|
||||
|
||||
public int UntrackedFiles { get; set; }
|
||||
@@ -614,6 +759,7 @@ public sealed class LibraryRebuilder(
|
||||
EntriesDeleted = EntriesDeleted,
|
||||
FilesVerified = FilesVerified,
|
||||
FilesMissing = FilesMissing,
|
||||
FilesUnreadable = FilesUnreadable,
|
||||
HashMismatches = HashMismatches,
|
||||
UntrackedFiles = UntrackedFiles,
|
||||
InvalidLines = InvalidLines,
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
using System.Security.Cryptography;
|
||||
using Febe.Core.Domain;
|
||||
using Febe.Core.Import;
|
||||
using Febe.Core.Library;
|
||||
using Febe.Tests.Import.Support;
|
||||
using Febe.Tests.Support;
|
||||
|
||||
namespace Febe.Tests.Import;
|
||||
|
||||
/// <summary>«Tieni il più grande» when the source file is larger: it replaces the library file (rule 12, spec §9.3).</summary>
|
||||
[TestClass]
|
||||
public sealed class ImportReplacementTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Two versions of the same photo in the source (the original and a smaller export), both paired with one smaller
|
||||
/// library file and both decided «Tieni il più grande»: only the largest takes its place, whatever the order of
|
||||
/// the files, and with «Sposta» the larger original is never lost.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
[DataRow(ImportMode.Copy, true)]
|
||||
[DataRow(ImportMode.Move, true)]
|
||||
[DataRow(ImportMode.Copy, false)]
|
||||
[DataRow(ImportMode.Move, false)]
|
||||
public async Task TwoSourceFilesPairedWithTheSameLibraryFileLeaveOnlyTheLargest(ImportMode mode, bool largestFirst)
|
||||
{
|
||||
using var temp = new TempFolder("importa");
|
||||
await using var bed = await ImportTestBed.CreateAsync(temp);
|
||||
var (marta, library) = await bed.AccountWithPhotoAsync("marta.demo", "60", ImportTestFiles.Resized(60, 800, 600, 400, 300, 70));
|
||||
var largest = ImportTestFiles.Photo(60, 800, 600, 92);
|
||||
var export = ImportTestFiles.Resized(60, 800, 600, 600, 450, 80);
|
||||
var largestPath = bed.Write(largestFirst ? "marta.demo/1-originale.jpg" : "marta.demo/2-originale.jpg", largest);
|
||||
var exportPath = bed.Write(largestFirst ? "marta.demo/2-esportata.jpg" : "marta.demo/1-esportata.jpg", export);
|
||||
var run = await bed.AnalyzeAsync();
|
||||
await bed.Review.SetModeAsync(run.Id, mode);
|
||||
|
||||
var cases = await bed.Review.ListDuplicatesAsync(run.Id);
|
||||
Assert.HasCount(2, cases);
|
||||
Assert.IsTrue(cases.All(c => c.Existing.MediaId == library.Media[0].Id), "Both source files are paired with the library file.");
|
||||
Assert.AreEqual(2, await bed.Review.DecideAsync(run.Id, cases[0].Pair.Id, DuplicateDecision.KeepLarger, DecisionScope.AllPending));
|
||||
var plan = await bed.Review.GetPlanAsync(run.Id);
|
||||
Assert.AreEqual(1, plan.ToReplace, "One library file, one replacement.");
|
||||
Assert.AreEqual(1, plan.ToImport);
|
||||
|
||||
await bed.Copier.CopyAsync(run.Id);
|
||||
|
||||
var media = (await bed.Library.Db.Media.GetAsync(library.Media[0].Id))!;
|
||||
var largestSha = Sha(largest);
|
||||
Assert.AreEqual(largestSha, media.Sha256, "The library keeps the largest version.");
|
||||
Assert.AreEqual((800, 600), (media.Width, media.Height));
|
||||
var onDisk = bed.Library.Layout.FullPath(marta, media.RelativePath!);
|
||||
Assert.AreEqual(largestSha, Sha(File.ReadAllBytes(onDisk)), "The record points to a file that exists and holds the largest version.");
|
||||
var folder = bed.Library.Layout.AccountFolder(marta);
|
||||
Assert.HasCount(1, Directory.GetFiles(folder, "*.jpg", SearchOption.AllDirectories), "One version in the library: no leftovers.");
|
||||
|
||||
var files = await bed.FilesAsync(run.Id);
|
||||
var winner = files.Single(f => f.Sha256 == largestSha);
|
||||
Assert.AreEqual(mode == ImportMode.Move ? ImportFileState.Moved : ImportFileState.Copied, winner.State, winner.LastError);
|
||||
StringAssert.StartsWith(winner.LastError, ImportText.NoteReplaced);
|
||||
Assert.AreEqual(media.RelativePath, winner.DestinationPath);
|
||||
var loser = files.Single(f => f.Sha256 == Sha(export));
|
||||
Assert.AreEqual(ImportFileState.Skipped, loser.State, "The smaller export never takes the place of the original.");
|
||||
Assert.AreEqual(ImportText.NoteLargerInSource, loser.LastError);
|
||||
|
||||
Assert.AreEqual(mode == ImportMode.Copy, File.Exists(largestPath), "With «Sposta» the original goes only once its copy is verified.");
|
||||
Assert.IsTrue(File.Exists(exportPath), "A skipped file is never deleted, not even with «Sposta».");
|
||||
CollectionAssert.AreEqual(mode == ImportMode.Move ? new[] { largestPath } : Array.Empty<string>(), bed.FileSystem.Deleted);
|
||||
|
||||
var journal = await AccountJournal.ReadAsync(folder);
|
||||
Assert.AreEqual(media.RelativePath, journal.Lines.Last(l => l.Entry.Id == "60").Files!.Single().Path);
|
||||
|
||||
// Running the copy again changes nothing.
|
||||
await bed.Copier.CopyAsync(run.Id);
|
||||
Assert.AreEqual(largestSha, (await bed.Library.Db.Media.GetAsync(media.Id))!.Sha256);
|
||||
Assert.HasCount(1, Directory.GetFiles(folder, "*.jpg", SearchOption.AllDirectories));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The plan compares sizes before the copy starts; the copy looks again at the file in the library right before
|
||||
/// replacing it, and a larger file that arrived meanwhile stays.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public async Task AReplacementNeverRemovesALargerFileThatArrivedMeanwhile()
|
||||
{
|
||||
using var temp = new TempFolder("importa");
|
||||
await using var bed = await ImportTestBed.CreateAsync(temp);
|
||||
var (marta, library) = await bed.AccountWithPhotoAsync("marta.demo", "62", ImportTestFiles.Resized(62, 800, 600, 400, 300, 70));
|
||||
bed.Write("marta.demo/a-altra.jpg", ImportTestFiles.Photo(63));
|
||||
var candidate = bed.Write("marta.demo/b-grande.jpg", ImportTestFiles.Photo(62, 800, 600, 92));
|
||||
var run = await bed.AnalyzeAsync();
|
||||
var pair = (await bed.Review.ListDuplicatesAsync(run.Id)).Single();
|
||||
await bed.Review.DecideAsync(run.Id, pair.Pair.Id, DuplicateDecision.KeepLarger);
|
||||
Assert.AreEqual(1, (await bed.Review.GetPlanAsync(run.Id)).ToReplace);
|
||||
|
||||
// Right after the first file, a larger version of the photo takes the place of the library file.
|
||||
var huge = ImportTestFiles.Photo(62, 1600, 1200, 92);
|
||||
var hugeRelative = "post/arrivata-nel-frattempo.jpg";
|
||||
var hugePath = bed.Library.Layout.FullPath(marta, hugeRelative);
|
||||
var swapped = false;
|
||||
var progress = new SyncProgress(p =>
|
||||
{
|
||||
if (swapped || p.Phase != ImportPhase.Copying || p.CurrentFile != "marta.demo/a-altra.jpg")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
swapped = true;
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(hugePath)!);
|
||||
File.WriteAllBytes(hugePath, huge);
|
||||
bed.Library.Db.Media.MarkStoredAsync(
|
||||
library.Media[0].Id,
|
||||
new StoredFile(hugeRelative, huge.Length, Sha(huge), null, 1600, 1200, null, false),
|
||||
MediaState.Downloaded,
|
||||
bed.Library.Clock.Now).GetAwaiter().GetResult();
|
||||
});
|
||||
|
||||
await bed.Copier.CopyAsync(run.Id, progress);
|
||||
|
||||
Assert.IsTrue(swapped);
|
||||
var media = (await bed.Library.Db.Media.GetAsync(library.Media[0].Id))!;
|
||||
Assert.AreEqual(Sha(huge), media.Sha256, "The larger file stays in the library.");
|
||||
Assert.IsTrue(File.Exists(hugePath));
|
||||
var file = await bed.FileAsync(run.Id, "marta.demo/b-grande.jpg");
|
||||
Assert.AreEqual(ImportFileState.Skipped, file.State);
|
||||
Assert.AreEqual(ImportText.NoteLibraryLarger, file.LastError);
|
||||
Assert.IsTrue(File.Exists(candidate));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An imported entry without Instagram id (with only a code, or with nothing but its file) whose file is replaced
|
||||
/// by a larger one: after losing the database the rebuild finds one entry with the new file, on every run.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
[DataRow(false)]
|
||||
[DataRow(true)]
|
||||
public async Task AReplacedImportedFileIsOneEntryAfterARebuild(bool withCode)
|
||||
{
|
||||
using var temp = new TempFolder("importa");
|
||||
await using var bed = await ImportTestBed.CreateAsync(temp);
|
||||
var marta = await bed.Library.Db.Accounts.AddAsync(Make.Account("marta.demo", "900"));
|
||||
var name = withCode ? "2026-10-10_18-22-05_DBx3kQ2_1.jpg" : "IMG_0001.jpg";
|
||||
var small = bed.Write("marta.demo/" + name, ImportTestFiles.Resized(64, 800, 600, 400, 300, 70));
|
||||
await bed.Copier.CopyAsync((await bed.AnalyzeAsync()).Id);
|
||||
var first = (await bed.Library.Db.Entries.ListByAccountAsync(marta.Id)).Single();
|
||||
Assert.AreEqual(withCode ? "DBx3kQ2" : null, first.Code);
|
||||
Assert.IsNull(first.InstagramId);
|
||||
|
||||
File.Delete(small);
|
||||
bed.Write("marta.demo/originali/" + name, ImportTestFiles.Photo(64, 800, 600, 92));
|
||||
var run = await bed.AnalyzeAsync();
|
||||
var pair = (await bed.Review.ListDuplicatesAsync(run.Id)).Single();
|
||||
await bed.Review.DecideAsync(run.Id, pair.Pair.Id, DuplicateDecision.KeepLarger);
|
||||
await bed.Copier.CopyAsync(run.Id);
|
||||
var replaced = (await bed.Library.Db.Media.ListByEntryAsync(first.Id)).Single();
|
||||
Assert.AreEqual((800, 600), (replaced.Width, replaced.Height), "Replaced by the larger file.");
|
||||
|
||||
await bed.Library.LoseDatabaseAsync();
|
||||
var rebuilder = new LibraryRebuilder(bed.Library.Db, bed.Library.Layout, bed.Library.Clock);
|
||||
var report = await rebuilder.RebuildAsync();
|
||||
|
||||
Assert.AreEqual(1, report.EntriesCreated, string.Join("; ", report.Notes));
|
||||
Assert.AreEqual(0, report.FilesMissing, "The replaced file is not reported as missing.");
|
||||
var account = (await bed.Library.Db.Accounts.FindByInstagramIdAsync("900"))!;
|
||||
var entry = (await bed.Library.Db.Entries.ListByAccountAsync(account.Id)).Single();
|
||||
Assert.AreEqual(replaced.RelativePath, (await bed.Library.Db.Media.ListByEntryAsync(entry.Id)).Single().RelativePath);
|
||||
Assert.AreEqual(0, (await rebuilder.RebuildAsync()).EntriesCreated, "A second run adds nothing.");
|
||||
Assert.HasCount(1, await bed.Library.Db.Entries.ListByAccountAsync(account.Id));
|
||||
}
|
||||
|
||||
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
|
||||
|
||||
private sealed class SyncProgress(Action<ImportProgress> report) : IProgress<ImportProgress>
|
||||
{
|
||||
public void Report(ImportProgress value) => report(value);
|
||||
}
|
||||
}
|
||||
@@ -89,6 +89,35 @@ public sealed class MediaReaderTests
|
||||
Assert.IsNull(Mp4Reader.Read(new MemoryStream("not a movie at all"u8.ToArray()), Now));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A 64-bit box size close to 2^63 overflows «offset + size»: the reader must stop on it, not walk backwards
|
||||
/// forever (the analysis would hang and every «Riprendi» would hit the same file again).
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public void AHugeSixtyFourBitBoxSizeEndsTheReadingInsteadOfLooping()
|
||||
{
|
||||
static byte[] Box(string type, params byte[][] parts) =>
|
||||
[.. BigEndian((uint)(8 + parts.Sum(p => p.Length))), .. Encoding.ASCII.GetBytes(type), .. parts.SelectMany(p => p)];
|
||||
static byte[] BigEndian(uint value) => [(byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte)value];
|
||||
byte[] huge = [0, 0, 0, 1, .. "xxxx"u8, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF8];
|
||||
var ftyp = Box("ftyp", "isom"u8.ToArray(), [0, 0, 2, 0]);
|
||||
|
||||
// Inside moov: a valid «free» box, then the damaged one.
|
||||
byte[] insideMoov = [.. ftyp, .. Box("moov", Box("free"), huge)];
|
||||
|
||||
// At the top level, after ftyp.
|
||||
byte[] topLevel = [.. ftyp, .. huge, .. new byte[32]];
|
||||
|
||||
foreach (var (name, bytes) in new[] { ("moov", insideMoov), ("primo livello", topLevel) })
|
||||
{
|
||||
var reading = Task.Run(() => Mp4Reader.Read(new MemoryStream(bytes), Now));
|
||||
Assert.IsTrue(reading.Wait(TimeSpan.FromSeconds(10)), $"{name}: la lettura deve finire.");
|
||||
var info = reading.Result;
|
||||
Assert.IsNull(info?.CreatedAt, name);
|
||||
Assert.IsNull(info?.Width, name);
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void AnMp4MadeByFfmpegCarriesItsCreationTime()
|
||||
{
|
||||
|
||||
@@ -52,9 +52,14 @@ public sealed class DailyBackupTests
|
||||
}
|
||||
|
||||
await File.WriteAllTextAsync(Path.Combine(folder, "appunti.txt"), "non toccare");
|
||||
await File.WriteAllTextAsync(Path.Combine(folder, "febe-2026-10-13.sqlite.abcd1234.part"), "copia interrotta");
|
||||
|
||||
var clock = new ManualClock(LibraryTimes.Noon(today));
|
||||
var interrupted = Path.Combine(folder, "febe-2026-10-12.sqlite.abcd1234.part");
|
||||
await File.WriteAllTextAsync(interrupted, "copia interrotta");
|
||||
File.SetLastWriteTimeUtc(interrupted, clock.Now.UtcDateTime.AddDays(-1));
|
||||
var writing = Path.Combine(folder, "febe-2026-10-13.sqlite.ef567890.part");
|
||||
await File.WriteAllTextAsync(writing, "copia in corso");
|
||||
File.SetLastWriteTimeUtc(writing, clock.Now.UtcDateTime.AddMinutes(-2));
|
||||
|
||||
var backup = new DailyBackup(db.Maintenance, clock, folder);
|
||||
var result = await backup.RunAsync();
|
||||
|
||||
@@ -64,9 +69,81 @@ public sealed class DailyBackupTests
|
||||
Assert.AreEqual(today, kept[0]);
|
||||
Assert.AreEqual(today.AddDays(-6), kept[^1]);
|
||||
Assert.HasCount(5, result.Removed); // four old copies and the interrupted one
|
||||
Assert.IsFalse(File.Exists(interrupted));
|
||||
Assert.IsTrue(File.Exists(writing), "A temporary copy written minutes ago may belong to a copy still running.");
|
||||
Assert.IsTrue(File.Exists(Path.Combine(folder, "appunti.txt")));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// «Fai una copia adesso» clicked twice, or clicked while the 3:20 copy runs: the copies take turns, and neither
|
||||
/// one removes the temporary file of the other.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public async Task TwoCopiesAtTheSameTimeBothSucceed()
|
||||
{
|
||||
using var temp = new TempFolder();
|
||||
var folder = temp.Combine("dati", "copie");
|
||||
var clock = new ManualClock(LibraryTimes.Noon(new DateOnly(2026, 10, 13)));
|
||||
var database = new SlowBackups();
|
||||
var first = new DailyBackup(database, clock, folder).RunNowAsync();
|
||||
await database.FirstStarted.Task.WaitAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
var second = new DailyBackup(database, clock, folder).RunNowAsync();
|
||||
await Task.WhenAny(second, Task.Delay(500)); // without turns, the second copy finishes here and its retention runs
|
||||
database.ReleaseFirst.SetResult();
|
||||
|
||||
var results = await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10));
|
||||
Assert.IsTrue(results.All(r => r.Created is not null));
|
||||
Assert.AreEqual(1, database.MaxConcurrent, "One copy of a folder at a time.");
|
||||
Assert.HasCount(1, Directory.GetFiles(folder, "*.sqlite"));
|
||||
Assert.IsEmpty(Directory.GetFiles(folder, "*.part"));
|
||||
}
|
||||
|
||||
/// <summary>VACUUM INTO stand-in: the first copy waits until the test lets it finish.</summary>
|
||||
private sealed class SlowBackups : Core.Storage.IDatabaseMaintenance
|
||||
{
|
||||
private int _calls;
|
||||
private int _running;
|
||||
private int _max;
|
||||
|
||||
public TaskCompletionSource FirstStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public TaskCompletionSource ReleaseFirst { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public int MaxConcurrent => Volatile.Read(ref _max);
|
||||
|
||||
public string DatabasePath => "febe.sqlite";
|
||||
|
||||
public int SchemaVersion => 1;
|
||||
|
||||
public async Task BackupToAsync(string destinationFile, CancellationToken ct = default)
|
||||
{
|
||||
var running = Interlocked.Increment(ref _running);
|
||||
int seen;
|
||||
while ((seen = Volatile.Read(ref _max)) < running && Interlocked.CompareExchange(ref _max, running, seen) != seen)
|
||||
{
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await File.WriteAllTextAsync(destinationFile, "copia", ct);
|
||||
if (Interlocked.Increment(ref _calls) == 1)
|
||||
{
|
||||
FirstStarted.SetResult();
|
||||
await ReleaseFirst.Task.WaitAsync(ct);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Decrement(ref _running);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<bool> CheckIntegrityAsync(CancellationToken ct = default) => Task.FromResult(true);
|
||||
|
||||
public Task OptimizeAsync(CancellationToken ct = default) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public void RecognizesOnlyItsOwnFileNames()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
using Febe.Core.Abstractions;
|
||||
using Febe.Core.Collection;
|
||||
using Febe.Core.Collection.Downloads;
|
||||
using Febe.Core.Domain;
|
||||
using Febe.Core.Import;
|
||||
using Febe.Core.Library;
|
||||
using Febe.Core.Storage;
|
||||
using Febe.Tests.Collection.Support;
|
||||
using Febe.Tests.Import.Support;
|
||||
using Febe.Tests.Support;
|
||||
|
||||
namespace Febe.Tests.Library;
|
||||
|
||||
/// <summary>
|
||||
/// Every saved entry gets its <c>_febe.jsonl</c> line (spec §8.5), also when the journal cannot be written at that moment
|
||||
/// (a sync client holding it, a network share that dropped) and when the last file of an entry to settle is not stored.
|
||||
/// A file already in the library is never treated as «nothing written».
|
||||
/// </summary>
|
||||
[TestClass]
|
||||
public sealed class JournalBacklogTests
|
||||
{
|
||||
[TestMethod]
|
||||
public async Task ADownloadedFileStaysInTheLibraryWhenTheJournalIsBusyAndItsLineFollows()
|
||||
{
|
||||
using var temp = new TempFolder("giornale");
|
||||
await using var lib = await TestLibrary.CreateAsync(temp);
|
||||
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
||||
await lib.SaveAsync(account, Make.Entry(account.Id, "prima", Make.T0.AddDays(-1)), [TestFile.Photo(1, 3)]);
|
||||
var entryId = await AddAsync(lib.Db, account, "p1", Make.Photo() with { SourceUrl = "https://cdn.esempio/1.jpg" });
|
||||
var queue = Queue(lib, temp, _ => (FetchOutcome.Completed, TestFile.PhotoBytes(21, 64, 80)));
|
||||
var journal = AccountJournal.PathFor(lib.Layout.AccountFolder(account));
|
||||
|
||||
DownloadReport report;
|
||||
await using (new FileStream(journal, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
|
||||
{
|
||||
report = await queue.RunOnceAsync(CancellationToken.None);
|
||||
}
|
||||
|
||||
Assert.AreEqual(1, report.Stored, report.ToString());
|
||||
var media = (await lib.Db.Media.ListByEntryAsync(entryId)).Single();
|
||||
Assert.AreEqual(MediaState.Downloaded, media.State, "The file is in the library: it is not to download again.");
|
||||
Assert.IsTrue(File.Exists(lib.Layout.FullPath(account, media.RelativePath!)));
|
||||
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.GetAsync(entryId))!.State, "It stays visible in the library.");
|
||||
Assert.AreEqual(entryId.ToString(System.Globalization.CultureInfo.InvariantCulture), await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
|
||||
Assert.IsTrue((await lib.Db.Events.ListAfterAsync(0)).Any(e => e.Code == "giornale-in-sospeso"));
|
||||
Assert.IsFalse((await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Any(l => l.Entry.Id == "p1"));
|
||||
|
||||
// The next pass writes the line, even with nothing to download.
|
||||
await queue.RunOnceAsync(CancellationToken.None);
|
||||
var line = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Single(l => l.Entry.Id == "p1");
|
||||
Assert.AreEqual(media.RelativePath, line.Files!.Single().Path);
|
||||
Assert.IsNull(await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
|
||||
|
||||
// So a rebuild after losing the database finds it, without downloading anything.
|
||||
await lib.LoseDatabaseAsync();
|
||||
await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock).RebuildAsync();
|
||||
var again = await lib.Db.Accounts.FindByInstagramIdAsync("1001");
|
||||
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.FindAsync(again!.Id, EntryKind.Post, "p1"))!.State);
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task AnEntryGetsItsLineAlsoWhenItsLastFileIsSkippedOrFails()
|
||||
{
|
||||
using var temp = new TempFolder("giornale");
|
||||
await using var lib = await TestLibrary.CreateAsync(temp);
|
||||
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
|
||||
var tooLarge = await AddAsync(lib.Db, account, "p2", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p2.jpg" }, Make.Video(2) with { SourceUrl = "https://cdn.esempio/grande.mp4" });
|
||||
var refused = await AddAsync(lib.Db, account, "p3", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p3.jpg" }, Make.Photo(2) with { SourceUrl = "https://cdn.esempio/altrove.jpg" });
|
||||
var queue = Queue(lib, temp, uri => uri.AbsolutePath switch
|
||||
{
|
||||
"/grande.mp4" => (FetchOutcome.Completed, new byte[5000]),
|
||||
"/altrove.jpg" => (FetchOutcome.HostNotAllowed, null),
|
||||
"/p2.jpg" => (FetchOutcome.Completed, TestFile.PhotoBytes(31, 64, 80)),
|
||||
_ => (FetchOutcome.Completed, TestFile.PhotoBytes(32, 64, 80)),
|
||||
});
|
||||
|
||||
var report = await queue.RunOnceAsync(CancellationToken.None);
|
||||
|
||||
Assert.AreEqual(2, report.Stored, report.ToString());
|
||||
Assert.AreEqual(1, report.Skipped);
|
||||
Assert.AreEqual(1, report.Failed);
|
||||
var lines = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines;
|
||||
foreach (var (id, entryId) in new[] { ("p2", tooLarge), ("p3", refused) })
|
||||
{
|
||||
var stored = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(m => m.Position == 1);
|
||||
var line = lines.SingleOrDefault(l => l.Entry.Id == id);
|
||||
Assert.IsNotNull(line, $"{id}: the entry is settled with a file in the library, so it has its line.");
|
||||
Assert.AreEqual(stored.RelativePath, line.Files!.Single().Path);
|
||||
}
|
||||
}
|
||||
|
||||
[TestMethod]
|
||||
public async Task AnImportedFileStaysCopiedWhenTheJournalIsBusyAndItsLineFollows()
|
||||
{
|
||||
using var temp = new TempFolder("giornale");
|
||||
await using var bed = await ImportTestBed.CreateAsync(temp);
|
||||
var (marta, _) = await bed.AccountWithPhotoAsync("marta.demo", "300", ImportTestFiles.Photo(30));
|
||||
bed.Write("marta.demo/nuova.jpg", ImportTestFiles.Photo(31));
|
||||
var run = await bed.AnalyzeAsync();
|
||||
var folder = bed.Library.Layout.AccountFolder(marta);
|
||||
|
||||
await using (new FileStream(AccountJournal.PathFor(folder), FileMode.Open, FileAccess.ReadWrite, FileShare.None))
|
||||
{
|
||||
await bed.Copier.CopyAsync(run.Id);
|
||||
}
|
||||
|
||||
var file = await bed.FileAsync(run.Id, "marta.demo/nuova.jpg");
|
||||
Assert.AreEqual(ImportFileState.Copied, file.State, file.LastError);
|
||||
Assert.IsNotNull(file.EntryId);
|
||||
Assert.AreEqual(MediaState.Imported, (await bed.Library.Db.Media.ListByEntryAsync(file.EntryId.Value)).Single().State);
|
||||
Assert.IsFalse((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
|
||||
|
||||
// Any later copy (or download pass) writes the line that is still missing.
|
||||
await bed.Copier.CopyAsync(run.Id);
|
||||
Assert.IsTrue((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
|
||||
Assert.IsNull(await bed.Library.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
|
||||
Assert.AreEqual(ImportFileState.Copied, (await bed.FileAsync(run.Id, "marta.demo/nuova.jpg")).State);
|
||||
}
|
||||
|
||||
private static DownloadQueue Queue(TestLibrary lib, TempFolder temp, Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) =>
|
||||
new(lib.Db, lib.Ingest, new TableFetcher(answer), new FakeDataBudget(), lib.Clock, new SeededRandom(1),
|
||||
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
||||
|
||||
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, params Media[] media)
|
||||
{
|
||||
var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1));
|
||||
var added = await db.Entries.AddAsync(entry, media.Select(m => m with { SourceSeenAt = Make.T0 }).ToList());
|
||||
return added.Stored.Entry.Id;
|
||||
}
|
||||
|
||||
/// <summary>A file fetcher answering from a table: no network at all.</summary>
|
||||
private sealed class TableFetcher(Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) : IFileFetcher
|
||||
{
|
||||
public async Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
|
||||
{
|
||||
var (outcome, bytes) = answer(address);
|
||||
if (bytes is null)
|
||||
{
|
||||
return new FetchResult(outcome, 0, null, "errore di prova");
|
||||
}
|
||||
|
||||
if (options?.MaxBytes is { } max && bytes.Length > max)
|
||||
{
|
||||
await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct);
|
||||
return new FetchResult(FetchOutcome.TooLarge, max, 200, "troppo grande");
|
||||
}
|
||||
|
||||
await destination.WriteAsync(bytes, ct);
|
||||
return new FetchResult(outcome, bytes.Length, 200, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
using Febe.Core.Domain;
|
||||
using Febe.Core.Library;
|
||||
using Febe.Tests.Support;
|
||||
|
||||
namespace Febe.Tests.Library;
|
||||
|
||||
/// <summary>
|
||||
/// «Ricostruisci il database dalla libreria» (spec §8.5) on libraries that changed after their files were saved: files
|
||||
/// removed or replaced by Alberto's choices, files that cannot be read. The rebuild respects the choices, never stops
|
||||
/// on one file, and a second run adds nothing.
|
||||
/// </summary>
|
||||
[TestClass]
|
||||
public sealed class LibraryRebuildRecoveryTests
|
||||
{
|
||||
/// <summary>One file of a series removed by «Tieni quello in libreria» stays removed, not «file mancante».</summary>
|
||||
[TestMethod]
|
||||
public async Task AFileOfASeriesRemovedByADuplicateDecisionStaysRemoved()
|
||||
{
|
||||
using var temp = new TempFolder("ricostruzione");
|
||||
await using var lib = await TestLibrary.CreateAsync(temp);
|
||||
var account = await lib.Db.Accounts.AddAsync(Make.Account("doppie.prova", "500"));
|
||||
await lib.SaveAsync(account, Make.Entry(account.Id, "100", Make.T0.AddDays(-3)), [TestFile.Photo(1, 41, 1080, 1350)]);
|
||||
var series = await lib.SaveAsync(account, Make.Entry(account.Id, "200", Make.T0), [TestFile.Photo(1, 41, 720, 900), TestFile.Photo(2, 77, 1080, 1080)]);
|
||||
var review = new DownloadDuplicateReview(lib.Db, lib.Layout, lib.Journal, lib.Clock, lib.Thumbnails, lib.Index);
|
||||
var pair = (await review.ListAsync()).Single(c => c.Candidate.EntryId == series.Entry.Id);
|
||||
Assert.AreEqual(1, (await review.DecideAsync(pair.Pair.Id, DuplicateDecision.KeepExisting)).FilesRemoved);
|
||||
var before = (await lib.Db.Entries.GetWithMediaAsync(series.Entry.Id))!;
|
||||
|
||||
await lib.LoseDatabaseAsync();
|
||||
var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails);
|
||||
var report = await rebuilder.RebuildAsync();
|
||||
|
||||
Assert.AreEqual(0, report.FilesMissing, string.Join("; ", report.Notes));
|
||||
var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("500"))!;
|
||||
var entry = (await lib.Db.Entries.FindAsync(accountAgain.Id, EntryKind.Post, "200"))!;
|
||||
var media = await lib.Db.Media.ListByEntryAsync(entry.Id);
|
||||
Assert.AreEqual(MediaState.Deleted, media.Single(m => m.Position == 1).State, "Alberto's removal is respected: never looked for or downloaded again.");
|
||||
Assert.AreEqual(MediaState.Downloaded, media.Single(m => m.Position == 2).State);
|
||||
Assert.AreEqual(before.Media.Single(m => m.Position == 2).RelativePath, media.Single(m => m.Position == 2).RelativePath);
|
||||
Assert.AreEqual(before.Entry.State, entry.State);
|
||||
|
||||
var again = await rebuilder.RebuildAsync();
|
||||
Assert.AreEqual(0, again.EntriesCreated);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A journal written before replacements closed the old file: an imported entry without identifier appears twice,
|
||||
/// once with the file that was replaced. Only the entry with its file comes back, on every run.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public async Task AnEntryWithoutIdentifierWhoseFileWasReplacedComesBackOnce()
|
||||
{
|
||||
using var temp = new TempFolder("ricostruzione");
|
||||
await using var lib = await TestLibrary.CreateAsync(temp);
|
||||
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "600"));
|
||||
var folder = lib.Layout.AccountFolder(account);
|
||||
var bytes = TestFile.PhotoBytes(5, 800, 600);
|
||||
Directory.CreateDirectory(Path.Combine(folder, "importati"));
|
||||
await File.WriteAllBytesAsync(Path.Combine(folder, "importati", "IMG_1 (2).jpg"), bytes);
|
||||
var entry = new Entry
|
||||
{
|
||||
AccountId = account.Id,
|
||||
Kind = EntryKind.Post,
|
||||
PublishedAt = Make.T0.AddDays(-400),
|
||||
Provenance = Provenance.Imported,
|
||||
DiscoveredAt = Make.T0,
|
||||
};
|
||||
Media Imported(string path, string sha) => Make.Photo() with { State = MediaState.Imported, RelativePath = path, Sha256 = sha, Bytes = 100 };
|
||||
await lib.Journal.AppendSavedAsync(folder, account, entry, [Imported("importati/IMG_1.jpg", Make.Sha(1))]);
|
||||
await lib.Journal.AppendSavedAsync(folder, account, entry, [Imported("importati/IMG_1 (2).jpg", Make.Sha(2))]);
|
||||
|
||||
await lib.LoseDatabaseAsync();
|
||||
var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock);
|
||||
var report = await rebuilder.RebuildAsync();
|
||||
|
||||
Assert.AreEqual(1, report.EntriesCreated);
|
||||
Assert.AreEqual(1, report.FilesMissing, "The old file is reported...");
|
||||
Assert.IsTrue(report.Notes.Any(n => n.Contains("importati/IMG_1.jpg", StringComparison.Ordinal) && n.Contains("non ripristinato", StringComparison.Ordinal)));
|
||||
var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("600"))!;
|
||||
var entries = await lib.Db.Entries.ListByAccountAsync(accountAgain.Id);
|
||||
Assert.HasCount(1, entries, "...but not restored as an entry of its own.");
|
||||
Assert.AreEqual("importati/IMG_1 (2).jpg", (await lib.Db.Media.ListByEntryAsync(entries[0].Id)).Single().RelativePath);
|
||||
|
||||
var again = await rebuilder.RebuildAsync();
|
||||
Assert.AreEqual(0, again.EntriesCreated, "A second run adds nothing.");
|
||||
Assert.HasCount(1, await lib.Db.Entries.ListByAccountAsync(accountAgain.Id));
|
||||
}
|
||||
|
||||
/// <summary>A remembered deletion of an imported entry without identifier is not added again by every run.</summary>
|
||||
[TestMethod]
|
||||
public async Task ARememberedDeletionWithoutIdentifierIsRestoredOnce()
|
||||
{
|
||||
using var temp = new TempFolder("ricostruzione");
|
||||
await using var lib = await TestLibrary.CreateAsync(temp);
|
||||
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "700"));
|
||||
var imported = await lib.SaveAsync(account, Make.Entry(account.Id, "x", Make.T0.AddDays(-300)) with { InstagramId = null, Code = null, Provenance = Provenance.Imported }, [TestFile.Photo(1, 9)]);
|
||||
await new LibraryDeletion(lib.Db, lib.Layout, lib.Journal, lib.Clock).DeleteAsync(imported.Entry.Id, DeletionMode.Remember);
|
||||
|
||||
await lib.LoseDatabaseAsync();
|
||||
var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock);
|
||||
Assert.AreEqual(1, (await rebuilder.RebuildAsync()).EntriesDeleted);
|
||||
var again = await rebuilder.RebuildAsync();
|
||||
|
||||
Assert.AreEqual(0, again.EntriesCreated);
|
||||
var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("700"))!;
|
||||
var entries = await lib.Db.Entries.ListByAccountAsync(accountAgain.Id);
|
||||
Assert.HasCount(1, entries);
|
||||
Assert.AreEqual(EntryState.Deleted, entries[0].State);
|
||||
Assert.AreEqual(imported.Media[0].Sha256, (await lib.Db.Media.ListByEntryAsync(entries[0].Id)).Single().Sha256, "The SHA-256 is remembered: an import does not bring it back.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A file that cannot be read (here: held exclusively by another program) and an unreadable journal do not stop
|
||||
/// the rebuild of everything else; the file stays linked as the journal recorded it.
|
||||
/// </summary>
|
||||
[TestMethod]
|
||||
public async Task AnUnreadableFileOrJournalDoesNotStopTheRebuild()
|
||||
{
|
||||
using var temp = new TempFolder("ricostruzione");
|
||||
await using var lib = await TestLibrary.CreateAsync(temp);
|
||||
var marta = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "800"));
|
||||
var luca = await lib.Db.Accounts.AddAsync(Make.Account("luca.demo", "801"));
|
||||
var zoe = await lib.Db.Accounts.AddAsync(Make.Account("zoe.demo", "802"));
|
||||
var locked = await lib.SaveAsync(marta, Make.Entry(marta.Id, "a1", Make.T0.AddDays(-2)), [TestFile.Photo(1, 11)]);
|
||||
var fine = await lib.SaveAsync(marta, Make.Entry(marta.Id, "a2", Make.T0.AddDays(-1)), [TestFile.Photo(1, 12)]);
|
||||
await lib.SaveAsync(luca, Make.Entry(luca.Id, "b1", Make.T0), [TestFile.Photo(1, 13)]);
|
||||
await lib.SaveAsync(zoe, Make.Entry(zoe.Id, "c1", Make.T0), [TestFile.Photo(1, 14)]);
|
||||
await lib.LoseDatabaseAsync();
|
||||
|
||||
var lockedPath = lib.Layout.FullPath(marta, locked.Media[0].RelativePath!);
|
||||
var report = await WhileHeldAsync(
|
||||
[lockedPath, AccountJournal.PathFor(lib.Layout.AccountFolder(luca))],
|
||||
() => new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails).RebuildAsync());
|
||||
|
||||
Assert.AreEqual(1, report.FilesUnreadable, string.Join("; ", report.Notes));
|
||||
Assert.AreEqual(2, report.FilesVerified, "The other files of the folder and of the following folders are verified.");
|
||||
Assert.IsTrue(report.Notes.Any(n => n.Contains("luca.demo", StringComparison.Ordinal) && n.Contains("_febe.jsonl", StringComparison.Ordinal)));
|
||||
var martaAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("800"))!;
|
||||
var restored = (await lib.Db.Media.ListByEntryAsync((await lib.Db.Entries.FindAsync(martaAgain.Id, EntryKind.Post, "a1"))!.Id)).Single();
|
||||
Assert.AreEqual(MediaState.Downloaded, restored.State, "The file is where the journal says: linked again, not to be downloaded.");
|
||||
Assert.AreEqual(locked.Media[0].RelativePath, restored.RelativePath);
|
||||
Assert.AreEqual(locked.Media[0].Sha256, restored.Sha256);
|
||||
StringAssert.Contains(restored.LastError, "illeggibile");
|
||||
Assert.IsNotNull(await lib.Db.Entries.FindAsync(martaAgain.Id, EntryKind.Post, "a2"));
|
||||
Assert.IsNotNull(await lib.Db.Accounts.FindByInstagramIdAsync("802"), "The folders after the unreadable one are rebuilt too.");
|
||||
Assert.IsNull(await lib.Db.Accounts.FindByInstagramIdAsync("801"), "The folder with the unreadable journal waits for the next run.");
|
||||
|
||||
var later = await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails).RebuildAsync();
|
||||
Assert.AreEqual(1, later.EntriesCreated, "Once readable, the skipped folder is rebuilt.");
|
||||
_ = fine;
|
||||
}
|
||||
|
||||
/// <summary>Runs <paramref name="action"/> while other handles hold the files with no sharing at all.</summary>
|
||||
private static async Task<T> WhileHeldAsync<T>(IReadOnlyList<string> paths, Func<Task<T>> action)
|
||||
{
|
||||
var handles = paths.Select(p => new FileStream(p, FileMode.Open, FileAccess.ReadWrite, FileShare.None)).ToList();
|
||||
try
|
||||
{
|
||||
return await action();
|
||||
}
|
||||
finally
|
||||
{
|
||||
foreach (var handle in handles)
|
||||
{
|
||||
await handle.DisposeAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user