diff --git a/src/Febe.Core/Collection/Downloads/DownloadQueue.cs b/src/Febe.Core/Collection/Downloads/DownloadQueue.cs index efc18bc..6571fc1 100644 --- a/src/Febe.Core/Collection/Downloads/DownloadQueue.cs +++ b/src/Febe.Core/Collection/Downloads/DownloadQueue.cs @@ -127,6 +127,7 @@ public sealed class DownloadQueue public async Task RunOnceAsync(CancellationToken ct) { Directory.CreateDirectory(_options.StagingFolder); + await _ingest.AppendPendingJournalAsync(ct).ConfigureAwait(false); // lines a busy _febe.jsonl refused earlier var tally = new Tally(); var attempted = new ConcurrentDictionary(); var accounts = new ConcurrentDictionary(); @@ -168,6 +169,15 @@ public sealed class DownloadQueue } private async Task ProcessAsync(PendingDownload pending, Tally tally, ConcurrentDictionary accounts, ConcurrentDictionary singlePostQueued, CancellationToken ct) + { + await ProcessFileAsync(pending, tally, accounts, singlePostQueued, ct).ConfigureAwait(false); + + // Whatever the outcome (stored, duplicate, skipped, failed), the entry may now be settled: its journal line is + // written once no file waits any more and at least one is in the library (spec §8.5). + await AppendJournalWhenSettledAsync(pending.Media.EntryId).ConfigureAwait(false); + } + + private async Task ProcessFileAsync(PendingDownload pending, Tally tally, ConcurrentDictionary accounts, ConcurrentDictionary singlePostQueued, CancellationToken ct) { var media = pending.Media; if (!accounts.TryGetValue(pending.AccountId, out var account)) @@ -319,7 +329,6 @@ public sealed class DownloadQueue { tally.Add(ref tally.Stored); Report(media, entry, DownloadStep.Stored); - await AppendJournalWhenCompleteAsync(entry, ct).ConfigureAwait(false); } else { @@ -393,12 +402,17 @@ public sealed class DownloadQueue Report(media, entry, DownloadStep.Expired); } - private async Task AppendJournalWhenCompleteAsync(Entry entry, CancellationToken ct) + /// + /// Appends the journal line of an entry once settled. A journal that cannot be written now (a sync client holding + /// _febe.jsonl, a share that dropped) never turns a stored file back into one to download: the line waits + /// in and is written by a later pass. + /// + private async Task AppendJournalWhenSettledAsync(long entryId) { - var files = await _storage.Media.ListByEntryAsync(entry.Id, ct).ConfigureAwait(false); - if (files.All(f => f.State is not (MediaState.ToDownload or MediaState.Expired))) + var files = await _storage.Media.ListByEntryAsync(entryId, CancellationToken.None).ConfigureAwait(false); + if (files.Any(f => EntryStateRules.IsPresent(f.State)) && files.All(f => f.State is not (MediaState.ToDownload or MediaState.Expired))) { - await _ingest.AppendJournalAsync(entry.Id, ct).ConfigureAwait(false); + await _ingest.TryAppendJournalAsync(entryId, CancellationToken.None).ConfigureAwait(false); } } diff --git a/src/Febe.Core/Import/ImportCopier.cs b/src/Febe.Core/Import/ImportCopier.cs index f039f06..0bae0c3 100644 --- a/src/Febe.Core/Import/ImportCopier.cs +++ b/src/Febe.Core/Import/ImportCopier.cs @@ -67,6 +67,7 @@ public sealed class ImportCopier(ImportEnvironment env) var files = await LoadFilesAsync(runId, ct).ConfigureAwait(false); var planner = await ImportPlanner.LoadAsync(env.Storage, runId, files, ct).ConfigureAwait(false); var ingest = new LibraryIngest(env.Storage, env.Layout, env.Naming, env.Journal, env.Clock, env.Thumbnails, hashIndex: null, env.SetFileTimeToPublication); + await ingest.AppendPendingJournalAsync(ct).ConfigureAwait(false); // lines a busy _febe.jsonl refused earlier var context = new CopyContext(run, layout, recognizers, ingest); var total = files.Count; var bytesTotal = files.Values.Sum(f => f.Bytes); @@ -99,6 +100,7 @@ public sealed class ImportCopier(ImportEnvironment env) progress?.Report(new ImportProgress(runId, ImportPhase.Copying, done, total, bytesDone, bytesTotal, file.RelativePath, new Dictionary(), copied, env.Clock.Now)); } + await ingest.AppendPendingJournalAsync(CancellationToken.None).ConfigureAwait(false); run = run with { State = ImportRunState.Completed, CompletedAt = env.Clock.Now, LastError = null }; await env.Storage.Imports.UpdateRunAsync(run, CancellationToken.None).ConfigureAwait(false); await env.Storage.Events.AppendAsync( @@ -226,15 +228,22 @@ public sealed class ImportCopier(ImportEnvironment env) catch (ContentChangedException) { tally.Failed++; - return await SaveAsync(file with { State = ImportFileState.Failed, LastError = ImportText.NoteChanged }).ConfigureAwait(false); + return await SaveAsync(await CurrentAsync(file).ConfigureAwait(false) with { State = ImportFileState.Failed, LastError = ImportText.NoteChanged }).ConfigureAwait(false); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { tally.Failed++; - return await SaveAsync(file with { State = ImportFileState.Failed, LastError = ImportAnalyzer.ErrorText(ex) }).ConfigureAwait(false); + return await SaveAsync(await CurrentAsync(file).ConfigureAwait(false) with { State = ImportFileState.Failed, LastError = ImportAnalyzer.ErrorText(ex) }).ConfigureAwait(false); } } + /// + /// The file's record as saved during the attempt (the entry it was given, so a resumed copy reuses it instead of + /// taking the file for one already in the library). + /// + private async Task CurrentAsync(ImportFile file) => + await env.Storage.Imports.GetFileAsync(file.Id, CancellationToken.None).ConfigureAwait(false) ?? file; + /// Copies one file into its destination account as an imported entry (or into an entry it completes). private async Task ImportAsync(CopyContext context, ImportFile file, Tally tally, CancellationToken ct) { @@ -267,7 +276,7 @@ public sealed class ImportCopier(ImportEnvironment env) if (EntryStateRules.IsPresent(media.State) && string.Equals(media.Sha256, file.Sha256, StringComparison.Ordinal) && media.RelativePath is { } already) { // Stored before an interruption: only the journal line and the state are missing. - await context.Ingest.AppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false); + await context.Ingest.TryAppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false); return await FinishAsync(context, file, account, entry.Id, already, null, tally).ConfigureAwait(false); } @@ -329,7 +338,7 @@ public sealed class ImportCopier(ImportEnvironment env) await env.Storage.Highlights.AddItemAsync(collection.Id, entry.Id, none).ConfigureAwait(false); } - await context.Ingest.AppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false); + await context.Ingest.TryAppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false); return await FinishAsync(context, file, account, entry.Id, result.File.RelativePath, null, tally).ConfigureAwait(false); } @@ -444,6 +453,14 @@ public sealed class ImportCopier(ImportEnvironment env) return await FinishAsync(context, file, (await AccountOfEntryAsync(media.EntryId, ct).ConfigureAwait(false)).Account, media.EntryId, media.RelativePath, ImportText.NoteReplaced, tally).ConfigureAwait(false); } + // Alberto chose «keep the larger»: the file in the library now is what counts, not the one seen when the copy + // was planned. A smaller or equal file never takes the place of a larger one. + if (!ImportPlanner.IsLarger(file.Width, file.Height, file.Bytes, media.Width, media.Height, media.Bytes)) + { + tally.Skipped++; + return await SaveAsync(file with { State = ImportFileState.Skipped, MatchedMediaId = media.Id, LastError = ImportText.NoteLibraryLarger }).ConfigureAwait(false); + } + var (account, entry) = await AccountOfEntryAsync(media.EntryId, ct).ConfigureAwait(false); var folder = env.Layout.AccountFolder(account); var source = LibraryLayout.FullPath(context.Layout.Root, file.RelativePath); @@ -473,9 +490,19 @@ public sealed class ImportCopier(ImportEnvironment env) ct).ConfigureAwait(false))!; } - // From here the replacement goes to the end: a half-done replacement is worse than a late one. + // From here the replacement goes to the end: a half-done replacement is worse than a late one. The new file is + // either registered or removed again: never left in the library with nothing pointing to it. var none = CancellationToken.None; - var onDisk = await env.FileSystem.ComputeSha256Async(saved.FullPath, none).ConfigureAwait(false); + string? onDisk = null; + try + { + onDisk = await env.FileSystem.ComputeSha256Async(saved.FullPath, none).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Not readable: not verified. + } + if (!string.Equals(onDisk, file.Sha256, StringComparison.Ordinal)) { TryDelete(saved.FullPath); // the new file was never registered: nothing in the library points to it @@ -488,14 +515,19 @@ public sealed class ImportCopier(ImportEnvironment env) var thumbnail = false; if (file.MediaKind == MediaKind.Photo) { - hash = PerceptualHash.ComputeFromFile(saved.FullPath); - await using (var stream = File.OpenRead(saved.FullPath)) + try { + hash = PerceptualHash.ComputeFromFile(saved.FullPath); + await using var stream = File.OpenRead(saved.FullPath); if (ImageDecoding.ReadSize(stream) is { } size) { (width, height) = size; } } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Verified a moment ago: recorded without perceptual hash, with the size measured by the analysis. + } thumbnail = TryThumbnail(saved.Sha256, () => File.OpenRead(saved.FullPath)); } @@ -513,7 +545,21 @@ public sealed class ImportCopier(ImportEnvironment env) media.State, env.Clock.Now, none).ConfigureAwait(false); - await context.Ingest.AppendJournalAsync(entry.Id, none).ConfigureAwait(false); + await context.Ingest.TryAppendJournalAsync(entry.Id, none).ConfigureAwait(false); + if (entry.InstagramId is null && string.IsNullOrWhiteSpace(entry.Code) && !string.Equals(oldRelative, saved.RelativePath, StringComparison.Ordinal)) + { + // An entry without identifier or code is known in the journal by its file: the line of the old file is + // closed («delete», not remembered), so a rebuild does not bring back an entry whose file is gone. + try + { + await env.Journal.AppendDeletedAsync(folder, account, entry, [oldRelative], remember: false, none).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Without this line a rebuild reports the old file as missing and does not restore it: nothing is lost. + } + } + if (hash is { } h) { env.HashIndex?.Add(media.Id, account.Id, h); @@ -538,7 +584,7 @@ public sealed class ImportCopier(ImportEnvironment env) { thumbnails.Delete(oldSha); } - catch (IOException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { // A thumbnail left behind costs a few kilobytes. } diff --git a/src/Febe.Core/Import/ImportPlanner.cs b/src/Febe.Core/Import/ImportPlanner.cs index bf3dbe8..27e2a3d 100644 --- a/src/Febe.Core/Import/ImportPlanner.cs +++ b/src/Febe.Core/Import/ImportPlanner.cs @@ -41,8 +41,10 @@ internal sealed record PlannedFile(ImportFile File, PlannedAction Action, string /// source; /// «Tieni quello in libreria»: the source file is skipped. «Tienili entrambi»: it is imported. «Tieni il più grande»: /// if the library file is larger the source file is skipped, otherwise it replaces the library file (more pixels win, -/// then more bytes; a tie keeps the library file). Inside the source, among files joined by «tieni il più grande» only -/// the largest is imported; +/// then more bytes; a tie keeps the library file). Several source files paired with the same library file form one +/// group: only the largest of them can replace it, the others are skipped, so a replacement never removes the copy +/// another replacement has just put in the library. Inside the source, among files joined by «tieni il più grande» +/// only the largest is imported; /// files of a folder without account stay out. /// /// @@ -88,6 +90,7 @@ internal sealed class ImportPlanner } var groups = new UnionFind(); + var rivals = new Dictionary>(); // library file → source files that may replace it foreach (var pair in pairs.Where(p => p.Decision != DuplicateDecision.Pending && p.CandidateImportFileId is not null)) { var candidate = pair.CandidateImportFileId!.Value; @@ -99,17 +102,13 @@ internal sealed class ImportPlanner planner.Lose(candidate, ImportText.NoteKeptExisting); break; case DuplicateDecision.KeepLarger: - var library = await storage.Media.GetAsync(mediaId, ct).ConfigureAwait(false); - if (library is not null && files.TryGetValue(candidate, out var file) - && IsLarger(file.Width, file.Height, file.Bytes, library.Width, library.Height, library.Bytes)) + if (!rivals.TryGetValue(mediaId, out var list)) { - planner._replacements[candidate] = mediaId; - } - else - { - planner.Lose(candidate, ImportText.NoteLibraryLarger); + list = []; + rivals[mediaId] = list; } + list.Add(candidate); break; default: break; // KeepBoth: imported @@ -134,20 +133,54 @@ internal sealed class ImportPlanner foreach (var group in groups.Groups()) { var members = group.Where(files.ContainsKey).Select(id => files[id]).ToList(); - var winner = members - .OrderByDescending(f => (long)(f.Width ?? 0) * (f.Height ?? 0)) - .ThenByDescending(f => f.Bytes) - .ThenBy(f => f.Id) - .FirstOrDefault(); + var winner = Largest(members); foreach (var member in members.Where(m => m.Id != winner?.Id)) { planner.Lose(member.Id, ImportText.NoteKeptOtherInSource); } } + // «Tieni il più grande» against a library file: the library file and every source file paired with it are one + // group, and only the largest of the group stays. Deciding pair by pair would let a second, smaller source + // file replace the copy the first one has just put in the library (and, with «Sposta», lose the larger one). + foreach (var (mediaId, candidates) in rivals) + { + var library = await storage.Media.GetAsync(mediaId, ct).ConfigureAwait(false); + // A file already skipped never entered the library: it does not compete. + var members = candidates.Distinct() + .Where(id => files.TryGetValue(id, out var f) && f.State != ImportFileState.Skipped && !planner._decided.ContainsKey(id)) + .Select(id => files[id]) + .ToList(); + var winner = Largest(members); + var replaces = winner is not null && library is not null + && (IsLarger(winner.Width, winner.Height, winner.Bytes, library.Width, library.Height, library.Bytes) + // The library already holds this very file: a replacement interrupted before its state was saved, + // which the copy finishes. + || string.Equals(winner.Sha256, library.Sha256, StringComparison.Ordinal)); + foreach (var member in members) + { + if (replaces && member.Id == winner!.Id) + { + planner._replacements[member.Id] = mediaId; + } + else + { + planner.Lose(member.Id, replaces ? ImportText.NoteLargerInSource : ImportText.NoteLibraryLarger); + } + } + } + return planner; } + /// The largest file: more pixels, then more bytes; on a tie the first one analysed. + private static ImportFile? Largest(IEnumerable files) => + files + .OrderByDescending(f => (long)(f.Width ?? 0) * (f.Height ?? 0)) + .ThenByDescending(f => f.Bytes) + .ThenBy(f => f.Id) + .FirstOrDefault(); + /// True when the folder of the file has (or will have) an account. public bool HasTarget(ImportFile file) => file.TargetAccountId is not null || _newAccounts.ContainsKey(file.Folder); diff --git a/src/Febe.Core/Import/ImportText.cs b/src/Febe.Core/Import/ImportText.cs index 3dab244..e16f137 100644 --- a/src/Febe.Core/Import/ImportText.cs +++ b/src/Febe.Core/Import/ImportText.cs @@ -136,6 +136,7 @@ public static class ImportText internal const string NoteKeptExisting = "Hai scelto di tenere il file già in libreria."; internal const string NoteLibraryLarger = "Hai scelto di tenere il più grande: è quello già in libreria."; internal const string NoteKeptOtherInSource = "Hai scelto di tenere l'altro file della sorgente."; + internal const string NoteLargerInSource = "Hai scelto di tenere il più grande: è un altro file della sorgente, che prende il posto di quello in libreria."; internal const string NoteNotRecognized = "Né foto né video: ignorato."; internal const string NoteAlreadyInLibrary = "Identico a un file già in libreria: saltato."; internal const string NoteReplaced = "Ha sostituito il file più piccolo della libreria, come hai scelto."; diff --git a/src/Febe.Core/Import/Mp4Reader.cs b/src/Febe.Core/Import/Mp4Reader.cs index 9a42453..adbe7d3 100644 --- a/src/Febe.Core/Import/Mp4Reader.cs +++ b/src/Febe.Core/Import/Mp4Reader.cs @@ -25,6 +25,9 @@ public static class Mp4Reader private const int MaxTopLevelBoxes = 10_000; + /// The most child boxes read inside one box (real files have a few dozen). + private const int MaxChildBoxes = 10_000; + /// Reads a file (read-only). Null when no moov box is found. /// The file. /// Current time: creation times after it (by more than a day) are ignored. @@ -72,7 +75,8 @@ public static class Mp4Reader size = length - position; // to the end of the file } - if (size < headerSize || position + size > length + 8) + // Compared without adding to the position: a 64-bit size close to 2^63 would overflow the sum and pass. + if (size < headerSize || size > length - position + 8) { return null; // damaged } @@ -193,7 +197,7 @@ public static class Mp4Reader { var list = new List<(string, int, int)>(); var offset = 0; - while (offset + 8 <= data.Length) + while (offset + 8 <= data.Length && list.Count < MaxChildBoxes) { long size = BinaryPrimitives.ReadUInt32BigEndian(data.Slice(offset, 4)); var type = System.Text.Encoding.Latin1.GetString(data.Slice(offset + 4, 4)); @@ -208,7 +212,9 @@ public static class Mp4Reader size = data.Length - offset; } - if (size < headerSize || offset + size > data.Length) + // «size > remaining» cannot overflow (unlike «offset + size»), and with «size >= headerSize» it keeps the + // offset moving forward: a damaged 64-bit size can neither loop back nor run past the end. + if (size < headerSize || size > data.Length - offset) { break; } diff --git a/src/Febe.Core/Library/DailyBackup.cs b/src/Febe.Core/Library/DailyBackup.cs index f14200c..2c4b613 100644 --- a/src/Febe.Core/Library/DailyBackup.cs +++ b/src/Febe.Core/Library/DailyBackup.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Globalization; using System.Text.RegularExpressions; using Febe.Core.Abstractions; @@ -18,6 +19,11 @@ public sealed record BackupResult(BackupFile? Created, IReadOnlyList Rem /// Daily copy of the database (spec §8.5): VACUUM INTO a new file Dati/copie/febe-YYYY-MM-DD.sqlite /// (local date), keeping seven days. A copy is written under a temporary name and renamed, so a cut copy never has a /// final name. Only files named febe-YYYY-MM-DD.sqlite are ever removed. +/// +/// The 3:20 loop and «Fai una copia adesso» each make their own instance: copies and retention of one folder take turns +/// (one gate per folder), and the retention removes only temporary copies left behind long ago, never one still being +/// written. +/// /// public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock clock, string backupsFolder, int keepDays = DailyBackup.DefaultKeepDays) { @@ -27,6 +33,12 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl private const string Extension = ".sqlite"; private const string PartSuffix = ".part"; + /// A temporary copy untouched for this long was left by a copy that stopped (crash, power cut). + private static readonly TimeSpan StalePart = TimeSpan.FromHours(6); + + private static readonly ConcurrentDictionary Gates = new( + OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); + [GeneratedRegex(@"^febe-(\d{4}-\d{2}-\d{2})\.sqlite$", RegexOptions.CultureInvariant)] private static partial Regex BackupName(); @@ -36,32 +48,39 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl /// File name of the copy of a day. public static string FileNameFor(DateOnly day) => Prefix + day.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) + Extension; + private SemaphoreSlim Gate => Gates.GetOrAdd(Folder, _ => new SemaphoreSlim(1, 1)); + /// Makes today's copy if missing, then applies the retention. Safe to call often. public async Task RunAsync(CancellationToken ct = default) { - Directory.CreateDirectory(Folder); - var today = LocalCalendar.Today(clock.Now, clock.LocalZone); - var target = Path.Combine(Folder, FileNameFor(today)); - BackupFile? created = null; - if (!File.Exists(target)) + await Gate.WaitAsync(ct).ConfigureAwait(false); + try { - var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix; - try + Directory.CreateDirectory(Folder); + var today = LocalCalendar.Today(clock.Now, clock.LocalZone); + var target = Path.Combine(Folder, FileNameFor(today)); + BackupFile? created = null; + if (!File.Exists(target)) { - await database.BackupToAsync(part, ct).ConfigureAwait(false); - File.Move(part, target, overwrite: false); - created = new BackupFile(today, target, new FileInfo(target).Length); - } - finally - { - if (File.Exists(part)) + var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix; + try { - File.Delete(part); + await database.BackupToAsync(part, ct).ConfigureAwait(false); + File.Move(part, target, overwrite: false); + created = new BackupFile(today, target, new FileInfo(target).Length); + } + finally + { + TryDelete(part); } } - } - return new BackupResult(created, ApplyRetention(today)); + return new BackupResult(created, Retain(today)); + } + finally + { + Gate.Release(); + } } /// @@ -71,26 +90,31 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl /// public async Task RunNowAsync(CancellationToken ct = default) { - Directory.CreateDirectory(Folder); - var today = LocalCalendar.Today(clock.Now, clock.LocalZone); - var target = Path.Combine(Folder, FileNameFor(today)); - var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix; - BackupFile created; + await Gate.WaitAsync(ct).ConfigureAwait(false); try { - await database.BackupToAsync(part, ct).ConfigureAwait(false); - File.Move(part, target, overwrite: true); - created = new BackupFile(today, target, new FileInfo(target).Length); + Directory.CreateDirectory(Folder); + var today = LocalCalendar.Today(clock.Now, clock.LocalZone); + var target = Path.Combine(Folder, FileNameFor(today)); + var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix; + BackupFile created; + try + { + await database.BackupToAsync(part, ct).ConfigureAwait(false); + File.Move(part, target, overwrite: true); + created = new BackupFile(today, target, new FileInfo(target).Length); + } + finally + { + TryDelete(part); + } + + return new BackupResult(created, Retain(today)); } finally { - if (File.Exists(part)) - { - File.Delete(part); - } + Gate.Release(); } - - return new BackupResult(created, ApplyRetention(today)); } /// The copies present, newest first. @@ -114,10 +138,25 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl } /// - /// Removes copies older than days (today included) and stale temporary copies. - /// Returns the removed paths. + /// Removes copies older than days (today included) and temporary copies left behind + /// by a copy that stopped (untouched for hours: one still being written is never removed). A file that cannot be + /// removed now (open in another program) is left for the next time. Returns the removed paths. /// public IReadOnlyList ApplyRetention(DateOnly today) + { + Gate.Wait(); + try + { + return Retain(today); + } + finally + { + Gate.Release(); + } + } + + /// The retention; the caller holds the gate. + private List Retain(DateOnly today) { var removed = new List(); if (!Directory.Exists(Folder)) @@ -128,19 +167,42 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl var oldestKept = today.AddDays(-(Math.Max(1, keepDays) - 1)); foreach (var backup in List().Where(b => b.Day < oldestKept)) { - File.Delete(backup.Path); - removed.Add(backup.Path); + if (TryDelete(backup.Path)) + { + removed.Add(backup.Path); + } } + var staleBefore = clock.Now.UtcDateTime - StalePart; foreach (var part in Directory.EnumerateFiles(Folder, Prefix + "*" + PartSuffix)) { - File.Delete(part); - removed.Add(part); + if (File.GetLastWriteTimeUtc(part) < staleBefore && TryDelete(part)) + { + removed.Add(part); + } } return removed; } + private static bool TryDelete(string path) + { + try + { + if (!File.Exists(path)) + { + return false; + } + + File.Delete(path); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; // the next retention tries again + } + } + /// The day of a backup file name, or null for any other file. public static DateOnly? TryParseDay(string fileName) { diff --git a/src/Febe.Core/Library/LibraryIngest.cs b/src/Febe.Core/Library/LibraryIngest.cs index d35a0b0..3eb2d3d 100644 --- a/src/Febe.Core/Library/LibraryIngest.cs +++ b/src/Febe.Core/Library/LibraryIngest.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Febe.Core.Abstractions; using Febe.Core.Domain; using Febe.Core.Storage; @@ -87,6 +88,14 @@ public sealed class LibraryIngest( bool setFileTimeToPublication = true, SafeFileWriter? writer = null) { + /// + /// Setting with the entries whose _febe.jsonl line is still to write (ids, comma separated): it survives a + /// restart, and downloads and imports share it. + /// + internal const string PendingJournalKey = "libreria.giornale-da-scrivere"; + + private static readonly SemaphoreSlim PendingJournalGate = new(1, 1); + private readonly SafeFileWriter _writer = writer ?? new SafeFileWriter(layout.MaxPathLength); private volatile LibraryNaming _naming = naming; private volatile bool _setFileTimeToPublication = setFileTimeToPublication; @@ -191,19 +200,27 @@ public sealed class LibraryIngest( return new MediaSaveResult(MediaSaveOutcome.IdenticalInAccount, null, identical, []); } + // From here the file is in the library under its final name: it is always recorded, even when reading it again + // for the perceptual hash and the size fails (callers treat an IOException as «nothing written»). ulong? hash = null; int? width = media.Width, height = media.Height; var hasThumbnail = false; if (media.Kind == MediaKind.Photo) { - hash = PerceptualHash.ComputeFromFile(saved.FullPath); - await using (var stream = File.OpenRead(saved.FullPath)) + try { + hash = PerceptualHash.ComputeFromFile(saved.FullPath); + await using var stream = File.OpenRead(saved.FullPath); if (ImageDecoding.ReadSize(stream) is { } size) { (width, height) = size; } } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Recorded without perceptual hash (no duplicate check by look for this file) and with the size the + // page gave; a rebuild of the database computes both again. + } hasThumbnail = TryThumbnail(saved.Sha256, () => File.OpenRead(saved.FullPath), owned: true); } @@ -261,6 +278,129 @@ public sealed class LibraryIngest( ct).ConfigureAwait(false); } + /// + /// Appends the _febe.jsonl line of an entry whose files are already in the library. When the journal cannot + /// be written now (held by a sync client, a network share that dropped, a full disk), the files stay recorded as + /// they are and the entry is remembered: writes its line later, so a rebuild + /// of the database still finds it (spec §8.5). Returns false in that case. + /// + public async Task TryAppendJournalAsync(long entryId, CancellationToken ct = default) + { + try + { + await AppendJournalAsync(entryId, ct).ConfigureAwait(false); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + await RememberPendingJournalAsync(entryId).ConfigureAwait(false); + return false; + } + } + + /// + /// Writes the journal lines that could not be written before (see ), each with the + /// entry as it is now. Entries deleted meanwhile, or left without files, need no line. Returns how many still wait. + /// + public async Task AppendPendingJournalAsync(CancellationToken ct = default) + { + await PendingJournalGate.WaitAsync(ct).ConfigureAwait(false); + try + { + var pending = await ReadPendingJournalAsync().ConfigureAwait(false); + if (pending.Count == 0) + { + return 0; + } + + var left = new List(); + foreach (var entryId in pending) + { + var stored = await storage.Entries.GetWithMediaAsync(entryId, CancellationToken.None).ConfigureAwait(false); + if (stored is null || stored.Entry.State == EntryState.Deleted || !stored.Media.Any(m => EntryStateRules.IsPresent(m.State))) + { + continue; + } + + if (ct.IsCancellationRequested) + { + left.Add(entryId); + continue; + } + + try + { + await AppendJournalAsync(entryId, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + left.Add(entryId); + } + } + + await WritePendingJournalAsync(left).ConfigureAwait(false); + return left.Count; + } + finally + { + PendingJournalGate.Release(); + } + } + + private async Task RememberPendingJournalAsync(long entryId) + { + await PendingJournalGate.WaitAsync(CancellationToken.None).ConfigureAwait(false); + try + { + var pending = await ReadPendingJournalAsync().ConfigureAwait(false); + if (pending.Contains(entryId)) + { + return; + } + + await WritePendingJournalAsync([.. pending, entryId]).ConfigureAwait(false); + if (pending.Count == 0) + { + var accountId = (await storage.Entries.GetAsync(entryId, CancellationToken.None).ConfigureAwait(false))?.AccountId; + await storage.Events.AppendAsync( + new EventRecord + { + At = clock.Now, + Code = "giornale-in-sospeso", + Severity = EventSeverity.Warning, + Message = "Non si è potuto scrivere _febe.jsonl nella cartella dell'account (file in uso da un altro programma o disco non raggiungibile). I file sono al sicuro in libreria: Febe scrive le righe mancanti appena può.", + AccountId = accountId, + }, + CancellationToken.None).ConfigureAwait(false); + } + } + finally + { + PendingJournalGate.Release(); + } + } + + private async Task> ReadPendingJournalAsync() + { + var text = await storage.Settings.GetAsync(PendingJournalKey, CancellationToken.None).ConfigureAwait(false); + var ids = new List(); + foreach (var part in (text ?? string.Empty).Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (long.TryParse(part, NumberStyles.Integer, CultureInfo.InvariantCulture, out var id) && !ids.Contains(id)) + { + ids.Add(id); + } + } + + return ids; + } + + private Task WritePendingJournalAsync(IReadOnlyList ids) => + storage.Settings.SetAsync( + PendingJournalKey, + ids.Count == 0 ? null : string.Join(',', ids.Select(id => id.ToString(CultureInfo.InvariantCulture))), + CancellationToken.None); + private bool TryThumbnail(string key, Func open, bool owned) { if (thumbnails is null) diff --git a/src/Febe.Core/Library/LibraryRebuilder.cs b/src/Febe.Core/Library/LibraryRebuilder.cs index 90afcea..1b59398 100644 --- a/src/Febe.Core/Library/LibraryRebuilder.cs +++ b/src/Febe.Core/Library/LibraryRebuilder.cs @@ -48,6 +48,12 @@ public sealed record RebuildReport public int FilesMissing { get; init; } + /// + /// Files that are on disk but could not be read (locked by another program, no permission, damaged disk): kept as + /// the journal describes them, without verification, and listed in . + /// + public int FilesUnreadable { get; init; } + /// Files whose SHA-256 differs from the journal (the file on disk wins). public int HashMismatches { get; init; } @@ -113,7 +119,7 @@ public sealed class LibraryRebuilder( { ct.ThrowIfCancellationRequested(); await RebuildFolderAsync(folders[i], options, report, ct).ConfigureAwait(false); - progress?.Report(new RebuildProgress(folders[i], i + 1, folders.Count, report.FilesVerified + report.FilesMissing)); + progress?.Report(new RebuildProgress(folders[i], i + 1, folders.Count, report.FilesVerified + report.FilesMissing + report.FilesUnreadable)); } report.AccountFolders = folders.Count; @@ -122,7 +128,18 @@ public sealed class LibraryRebuilder( private async Task RebuildFolderAsync(string folder, RebuildOptions options, Accumulator report, CancellationToken ct) { - var journal = await AccountJournal.ReadAsync(folder, ct).ConfigureAwait(false); + JournalContent journal; + try + { + journal = await AccountJournal.ReadAsync(folder, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // One unreadable journal does not stop the others: this folder can be rebuilt again later. + report.Note($"«{Path.GetFileName(folder)}»: _febe.jsonl non si può leggere ({Reason(ex)}); la cartella non è stata ricostruita."); + return; + } + report.InvalidLines += journal.InvalidLines; if (journal.Lines.Count == 0) { @@ -157,8 +174,26 @@ public sealed class LibraryRebuilder( foreach (var item in items) { ct.ThrowIfCancellationRequested(); + if (!item.Deleted && !item.HasIdentity && item.Files.Count > 0 && item.Files.Keys.All(n => checks[(item, n)].IsMissing)) + { + // An entry without identifier or code whose files are all gone has nothing to show and nothing to download + // again: it is reported, not restored. (A journal written before file replacements recorded the old path + // as deleted holds such an entry for every imported file replaced by «Tieni il più grande».) + foreach (var file in item.Files.Values) + { + report.FilesMissing++; + report.Note($"File mancante: {file.Path} (elemento senza identificativo, non ripristinato)"); + } + + continue; + } + var entry = BuildEntry(account.Id, item); - var media = item.Files.Values.OrderBy(f => f.N).Select(f => BuildMedia(item, f, item.Deleted ? null : checks[(item, f.N)], report)).ToList(); + var media = item.Files.Values + .Select(f => BuildMedia(item, f, item.Deleted ? null : checks[(item, f.N)], report)) + .Concat(item.Removed.Values.Select(BuildRemoved)) + .OrderBy(m => m.Position) + .ToList(); batch.Add((item, new NewEntry(entry, media))); if (batch.Count >= BatchSize) { @@ -192,7 +227,8 @@ public sealed class LibraryRebuilder( { var present = item.InstagramId is { } id ? known.Contains((item.Kind, id)) - : item.Files.Values.Any(f => paths.ContainsKey(LibraryLayout.ToStored(f.Path))); + : item.Files.Values.Any(f => paths.ContainsKey(LibraryLayout.ToStored(f.Path))) + || (item.Deleted && await IsDeletionKnownAsync(account, item, ct).ConfigureAwait(false)); if (!present) { missing.Add(item); @@ -216,6 +252,24 @@ public sealed class LibraryRebuilder( return missing; } + /// + /// A remembered deletion of an entry without identifier has no file path left in the database: it is recognised by + /// the SHA-256 of its files, so a second rebuild does not add it again. + /// + private async Task IsDeletionKnownAsync(Account account, Item item, CancellationToken ct) + { + foreach (var sha in item.Files.Values.Select(f => f.Sha256).OfType().Where(s => s.Length == 64)) + { + var same = await storage.Media.FindBySha256Async(sha.ToLowerInvariant(), ct).ConfigureAwait(false); + if (same.Any(s => s.Entry.AccountId == account.Id && s.Entry.Kind == item.Kind && s.Entry.State == EntryState.Deleted)) + { + return true; + } + } + + return false; + } + private async Task ResolveAccountAsync(string folder, IReadOnlyList lines, Accumulator report, CancellationToken ct) { var last = lines[^1].Account; @@ -255,7 +309,11 @@ public sealed class LibraryRebuilder( return await storage.Accounts.AddAsync(created, ct).ConfigureAwait(false); } - /// Replays the journal: the last save of an entry wins, deletions remove it or mark it. + /// + /// Replays the journal: the last save of an entry wins, deletions remove it or mark it. Every save line lists all + /// the files present at that moment, so a position a later save no longer lists was removed (one file of a series + /// dropped by a duplicate decision): it comes back as «eliminato», never as a file to find or download again. + /// private static List Fold(IReadOnlyList lines, Accumulator report) { var order = new List(); @@ -286,11 +344,20 @@ public sealed class LibraryRebuilder( else { item.Last = line; + if (line.Files is { Count: > 0 } listed) + { + foreach (var dropped in item.Files.Keys.Where(n => !listed.Any(f => f.N == n)).ToList()) + { + item.Removed[dropped] = item.Files[dropped]; + item.Files.Remove(dropped); + } + } } foreach (var file in line.Files ?? []) { item.Files[file.N] = file; + item.Removed.Remove(file.N); } } else if (line.Remember == true) @@ -315,6 +382,10 @@ public sealed class LibraryRebuilder( return order; } + /// + /// The identity of an entry across its lines: the Instagram id; else the code (an imported entry known by its code + /// keeps it when one of its files is replaced); else the first file, for entries without either. + /// private static string KeyOf(EntryKind kind, JournalLine line) { if (!string.IsNullOrWhiteSpace(line.Entry.Id)) @@ -322,6 +393,11 @@ public sealed class LibraryRebuilder( return $"{kind}|id|{line.Entry.Id.Trim()}"; } + if (!string.IsNullOrWhiteSpace(line.Entry.Code)) + { + return $"{kind}|code|{line.Entry.Code.Trim()}"; + } + var firstPath = line.Files?.OrderBy(f => f.N).FirstOrDefault()?.Path; return firstPath is not null ? $"{kind}|path|{firstPath}" @@ -345,47 +421,71 @@ public sealed class LibraryRebuilder( return FileCheck.Missing; } + // A file that cannot be read (locked by another program, no permission, bad sectors, a cloud placeholder that + // cannot be fetched offline) must not stop the rebuild of everything else. string sha; long bytes; - await using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 128 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan)) + try { + await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 128 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan); bytes = stream.Length; sha = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, ct).ConfigureAwait(false)); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + var recorded = file.Sha256 is { Length: 64 } s ? s.ToLowerInvariant() : null; + return FileCheck.CannotRead(Reason(ex)) with { HasThumbnail = recorded is not null && (thumbnails?.Exists(recorded) ?? false) }; + } var isVideo = string.Equals(file.Type, "video", StringComparison.OrdinalIgnoreCase); ulong? phash = null; int? width = file.Width, height = file.Height; var thumbnail = thumbnails?.Exists(sha) ?? false; - if (!isVideo) + try { - phash = PerceptualHash.ComputeFromFile(path); - await using (var stream = File.OpenRead(path)) + if (!isVideo) { - if (ImageDecoding.ReadSize(stream) is { } size) + phash = PerceptualHash.ComputeFromFile(path); + await using (var stream = File.OpenRead(path)) { - (width, height) = size; + if (ImageDecoding.ReadSize(stream) is { } size) + { + (width, height) = size; + } } } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Hashed but not decoded a second time: the file stays, without perceptual hash. + } - if (createThumbnails && thumbnails is not null && !thumbnail) + try + { + if (!isVideo && createThumbnails && thumbnails is not null && !thumbnail) { thumbnail = thumbnails.CreateFromFile(sha, path); } - } - else if (createThumbnails && thumbnails is not null && !thumbnail && frames is not null) - { - var frame = await frames.ExtractJpegAsync(path, ct).ConfigureAwait(false); - if (frame is not null) + else if (isVideo && createThumbnails && thumbnails is not null && !thumbnail && frames is not null) { - using var stream = new MemoryStream(frame); - thumbnail = thumbnails.Create(sha, stream); + var frame = await frames.ExtractJpegAsync(path, ct).ConfigureAwait(false); + if (frame is not null) + { + using var stream = new MemoryStream(frame); + thumbnail = thumbnails.Create(sha, stream); + } } } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + thumbnail = false; // the neutral tile until a thumbnail can be made + } return new FileCheck(true, bytes, sha, phash, width, height, thumbnail); } + private static string Reason(Exception ex) => ex is UnauthorizedAccessException ? "permesso negato" : ex.Message; + private static Entry BuildEntry(long accountId, Item item) { var e = item.Last.Entry; @@ -429,6 +529,23 @@ public sealed class LibraryRebuilder( return media with { State = MediaState.Deleted }; } + var imported = string.Equals(item.Last.Entry.Provenance, "imported", StringComparison.OrdinalIgnoreCase); + if (check is { Unreadable: { } reason }) + { + // The file is where the journal says: it stays linked as recorded there (nothing to download again), and + // the report says it could not be verified. + report.FilesUnreadable++; + report.Note($"File illeggibile, ripristinato senza verifica: {file.Path} ({reason})"); + return media with + { + State = imported ? MediaState.Imported : MediaState.Downloaded, + RelativePath = LibraryLayout.ToStored(file.Path), + HasThumbnail = check.HasThumbnail, + DownloadedAt = item.Last.At, + LastError = $"File illeggibile durante la ricostruzione del database: {reason}", + }; + } + if (check is not { Exists: true }) { report.FilesMissing++; @@ -443,7 +560,6 @@ public sealed class LibraryRebuilder( report.Note($"Contenuto diverso da quello registrato: {file.Path}"); } - var imported = string.Equals(item.Last.Entry.Provenance, "imported", StringComparison.OrdinalIgnoreCase); return media with { State = imported ? MediaState.Imported : MediaState.Downloaded, @@ -523,10 +639,24 @@ public sealed class LibraryRebuilder( batch.Clear(); } + /// A file of a series removed by Alberto after it was saved: known, settled, never downloaded again. + private static Media BuildRemoved(JournalFile file) => new() + { + Position = file.N, + Kind = string.Equals(file.Type, "video", StringComparison.OrdinalIgnoreCase) ? MediaKind.Video : MediaKind.Photo, + InstagramId = file.Id, + Width = file.Width, + Height = file.Height, + Duration = file.DurationMs is { } ms ? TimeSpan.FromMilliseconds(ms) : null, + Sha256 = file.Sha256 is { Length: 64 } s ? s.ToLowerInvariant() : null, + Bytes = file.Bytes, + State = MediaState.Deleted, + }; + private static void CountUntracked(string folder, List items, Accumulator report) { var known = new HashSet( - items.SelectMany(i => i.Files.Values).Select(f => LibraryLayout.ToStored(f.Path)), + items.SelectMany(i => i.Files.Values.Concat(i.Removed.Values)).Select(f => LibraryLayout.ToStored(f.Path)), OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal); foreach (var file in Directory.EnumerateFiles(folder, "*", new EnumerationOptions { RecurseSubdirectories = true, IgnoreInaccessible = true })) { @@ -548,6 +678,13 @@ public sealed class LibraryRebuilder( private sealed record FileCheck(bool Exists, long Bytes, string Sha256, ulong? PerceptualHash, int? Width, int? Height, bool HasThumbnail) { public static FileCheck Missing { get; } = new(false, 0, string.Empty, null, null, null, false); + + /// Set when the file is on disk but could not be read: why. + public string? Unreadable { get; init; } + + public bool IsMissing => !Exists && Unreadable is null; + + public static FileCheck CannotRead(string reason) => Missing with { Unreadable = reason }; } private sealed class Item(EntryKind kind, JournalLine first) @@ -560,8 +697,14 @@ public sealed class LibraryRebuilder( public string? InstagramId => string.IsNullOrWhiteSpace(Last.Entry.Id) ? null : Last.Entry.Id.Trim(); + /// An Instagram id or a code: the entry is more than its files. + public bool HasIdentity => InstagramId is not null || !string.IsNullOrWhiteSpace(Last.Entry.Code); + public SortedDictionary Files { get; } = []; + /// Positions listed by an earlier save line and no longer by the last one: removed by Alberto. + public SortedDictionary Removed { get; } = []; + public bool Deleted { get; set; } public DateTimeOffset? DeletedAt { get; set; } @@ -589,6 +732,8 @@ public sealed class LibraryRebuilder( public int FilesMissing { get; set; } + public int FilesUnreadable { get; set; } + public int HashMismatches { get; set; } public int UntrackedFiles { get; set; } @@ -614,6 +759,7 @@ public sealed class LibraryRebuilder( EntriesDeleted = EntriesDeleted, FilesVerified = FilesVerified, FilesMissing = FilesMissing, + FilesUnreadable = FilesUnreadable, HashMismatches = HashMismatches, UntrackedFiles = UntrackedFiles, InvalidLines = InvalidLines, diff --git a/tests/Febe.Tests/Import/ImportReplacementTests.cs b/tests/Febe.Tests/Import/ImportReplacementTests.cs new file mode 100644 index 0000000..ebfd977 --- /dev/null +++ b/tests/Febe.Tests/Import/ImportReplacementTests.cs @@ -0,0 +1,175 @@ +using System.Security.Cryptography; +using Febe.Core.Domain; +using Febe.Core.Import; +using Febe.Core.Library; +using Febe.Tests.Import.Support; +using Febe.Tests.Support; + +namespace Febe.Tests.Import; + +/// «Tieni il più grande» when the source file is larger: it replaces the library file (rule 12, spec §9.3). +[TestClass] +public sealed class ImportReplacementTests +{ + /// + /// Two versions of the same photo in the source (the original and a smaller export), both paired with one smaller + /// library file and both decided «Tieni il più grande»: only the largest takes its place, whatever the order of + /// the files, and with «Sposta» the larger original is never lost. + /// + [TestMethod] + [DataRow(ImportMode.Copy, true)] + [DataRow(ImportMode.Move, true)] + [DataRow(ImportMode.Copy, false)] + [DataRow(ImportMode.Move, false)] + public async Task TwoSourceFilesPairedWithTheSameLibraryFileLeaveOnlyTheLargest(ImportMode mode, bool largestFirst) + { + using var temp = new TempFolder("importa"); + await using var bed = await ImportTestBed.CreateAsync(temp); + var (marta, library) = await bed.AccountWithPhotoAsync("marta.demo", "60", ImportTestFiles.Resized(60, 800, 600, 400, 300, 70)); + var largest = ImportTestFiles.Photo(60, 800, 600, 92); + var export = ImportTestFiles.Resized(60, 800, 600, 600, 450, 80); + var largestPath = bed.Write(largestFirst ? "marta.demo/1-originale.jpg" : "marta.demo/2-originale.jpg", largest); + var exportPath = bed.Write(largestFirst ? "marta.demo/2-esportata.jpg" : "marta.demo/1-esportata.jpg", export); + var run = await bed.AnalyzeAsync(); + await bed.Review.SetModeAsync(run.Id, mode); + + var cases = await bed.Review.ListDuplicatesAsync(run.Id); + Assert.HasCount(2, cases); + Assert.IsTrue(cases.All(c => c.Existing.MediaId == library.Media[0].Id), "Both source files are paired with the library file."); + Assert.AreEqual(2, await bed.Review.DecideAsync(run.Id, cases[0].Pair.Id, DuplicateDecision.KeepLarger, DecisionScope.AllPending)); + var plan = await bed.Review.GetPlanAsync(run.Id); + Assert.AreEqual(1, plan.ToReplace, "One library file, one replacement."); + Assert.AreEqual(1, plan.ToImport); + + await bed.Copier.CopyAsync(run.Id); + + var media = (await bed.Library.Db.Media.GetAsync(library.Media[0].Id))!; + var largestSha = Sha(largest); + Assert.AreEqual(largestSha, media.Sha256, "The library keeps the largest version."); + Assert.AreEqual((800, 600), (media.Width, media.Height)); + var onDisk = bed.Library.Layout.FullPath(marta, media.RelativePath!); + Assert.AreEqual(largestSha, Sha(File.ReadAllBytes(onDisk)), "The record points to a file that exists and holds the largest version."); + var folder = bed.Library.Layout.AccountFolder(marta); + Assert.HasCount(1, Directory.GetFiles(folder, "*.jpg", SearchOption.AllDirectories), "One version in the library: no leftovers."); + + var files = await bed.FilesAsync(run.Id); + var winner = files.Single(f => f.Sha256 == largestSha); + Assert.AreEqual(mode == ImportMode.Move ? ImportFileState.Moved : ImportFileState.Copied, winner.State, winner.LastError); + StringAssert.StartsWith(winner.LastError, ImportText.NoteReplaced); + Assert.AreEqual(media.RelativePath, winner.DestinationPath); + var loser = files.Single(f => f.Sha256 == Sha(export)); + Assert.AreEqual(ImportFileState.Skipped, loser.State, "The smaller export never takes the place of the original."); + Assert.AreEqual(ImportText.NoteLargerInSource, loser.LastError); + + Assert.AreEqual(mode == ImportMode.Copy, File.Exists(largestPath), "With «Sposta» the original goes only once its copy is verified."); + Assert.IsTrue(File.Exists(exportPath), "A skipped file is never deleted, not even with «Sposta»."); + CollectionAssert.AreEqual(mode == ImportMode.Move ? new[] { largestPath } : Array.Empty(), bed.FileSystem.Deleted); + + var journal = await AccountJournal.ReadAsync(folder); + Assert.AreEqual(media.RelativePath, journal.Lines.Last(l => l.Entry.Id == "60").Files!.Single().Path); + + // Running the copy again changes nothing. + await bed.Copier.CopyAsync(run.Id); + Assert.AreEqual(largestSha, (await bed.Library.Db.Media.GetAsync(media.Id))!.Sha256); + Assert.HasCount(1, Directory.GetFiles(folder, "*.jpg", SearchOption.AllDirectories)); + } + + /// + /// The plan compares sizes before the copy starts; the copy looks again at the file in the library right before + /// replacing it, and a larger file that arrived meanwhile stays. + /// + [TestMethod] + public async Task AReplacementNeverRemovesALargerFileThatArrivedMeanwhile() + { + using var temp = new TempFolder("importa"); + await using var bed = await ImportTestBed.CreateAsync(temp); + var (marta, library) = await bed.AccountWithPhotoAsync("marta.demo", "62", ImportTestFiles.Resized(62, 800, 600, 400, 300, 70)); + bed.Write("marta.demo/a-altra.jpg", ImportTestFiles.Photo(63)); + var candidate = bed.Write("marta.demo/b-grande.jpg", ImportTestFiles.Photo(62, 800, 600, 92)); + var run = await bed.AnalyzeAsync(); + var pair = (await bed.Review.ListDuplicatesAsync(run.Id)).Single(); + await bed.Review.DecideAsync(run.Id, pair.Pair.Id, DuplicateDecision.KeepLarger); + Assert.AreEqual(1, (await bed.Review.GetPlanAsync(run.Id)).ToReplace); + + // Right after the first file, a larger version of the photo takes the place of the library file. + var huge = ImportTestFiles.Photo(62, 1600, 1200, 92); + var hugeRelative = "post/arrivata-nel-frattempo.jpg"; + var hugePath = bed.Library.Layout.FullPath(marta, hugeRelative); + var swapped = false; + var progress = new SyncProgress(p => + { + if (swapped || p.Phase != ImportPhase.Copying || p.CurrentFile != "marta.demo/a-altra.jpg") + { + return; + } + + swapped = true; + Directory.CreateDirectory(Path.GetDirectoryName(hugePath)!); + File.WriteAllBytes(hugePath, huge); + bed.Library.Db.Media.MarkStoredAsync( + library.Media[0].Id, + new StoredFile(hugeRelative, huge.Length, Sha(huge), null, 1600, 1200, null, false), + MediaState.Downloaded, + bed.Library.Clock.Now).GetAwaiter().GetResult(); + }); + + await bed.Copier.CopyAsync(run.Id, progress); + + Assert.IsTrue(swapped); + var media = (await bed.Library.Db.Media.GetAsync(library.Media[0].Id))!; + Assert.AreEqual(Sha(huge), media.Sha256, "The larger file stays in the library."); + Assert.IsTrue(File.Exists(hugePath)); + var file = await bed.FileAsync(run.Id, "marta.demo/b-grande.jpg"); + Assert.AreEqual(ImportFileState.Skipped, file.State); + Assert.AreEqual(ImportText.NoteLibraryLarger, file.LastError); + Assert.IsTrue(File.Exists(candidate)); + } + + /// + /// An imported entry without Instagram id (with only a code, or with nothing but its file) whose file is replaced + /// by a larger one: after losing the database the rebuild finds one entry with the new file, on every run. + /// + [TestMethod] + [DataRow(false)] + [DataRow(true)] + public async Task AReplacedImportedFileIsOneEntryAfterARebuild(bool withCode) + { + using var temp = new TempFolder("importa"); + await using var bed = await ImportTestBed.CreateAsync(temp); + var marta = await bed.Library.Db.Accounts.AddAsync(Make.Account("marta.demo", "900")); + var name = withCode ? "2026-10-10_18-22-05_DBx3kQ2_1.jpg" : "IMG_0001.jpg"; + var small = bed.Write("marta.demo/" + name, ImportTestFiles.Resized(64, 800, 600, 400, 300, 70)); + await bed.Copier.CopyAsync((await bed.AnalyzeAsync()).Id); + var first = (await bed.Library.Db.Entries.ListByAccountAsync(marta.Id)).Single(); + Assert.AreEqual(withCode ? "DBx3kQ2" : null, first.Code); + Assert.IsNull(first.InstagramId); + + File.Delete(small); + bed.Write("marta.demo/originali/" + name, ImportTestFiles.Photo(64, 800, 600, 92)); + var run = await bed.AnalyzeAsync(); + var pair = (await bed.Review.ListDuplicatesAsync(run.Id)).Single(); + await bed.Review.DecideAsync(run.Id, pair.Pair.Id, DuplicateDecision.KeepLarger); + await bed.Copier.CopyAsync(run.Id); + var replaced = (await bed.Library.Db.Media.ListByEntryAsync(first.Id)).Single(); + Assert.AreEqual((800, 600), (replaced.Width, replaced.Height), "Replaced by the larger file."); + + await bed.Library.LoseDatabaseAsync(); + var rebuilder = new LibraryRebuilder(bed.Library.Db, bed.Library.Layout, bed.Library.Clock); + var report = await rebuilder.RebuildAsync(); + + Assert.AreEqual(1, report.EntriesCreated, string.Join("; ", report.Notes)); + Assert.AreEqual(0, report.FilesMissing, "The replaced file is not reported as missing."); + var account = (await bed.Library.Db.Accounts.FindByInstagramIdAsync("900"))!; + var entry = (await bed.Library.Db.Entries.ListByAccountAsync(account.Id)).Single(); + Assert.AreEqual(replaced.RelativePath, (await bed.Library.Db.Media.ListByEntryAsync(entry.Id)).Single().RelativePath); + Assert.AreEqual(0, (await rebuilder.RebuildAsync()).EntriesCreated, "A second run adds nothing."); + Assert.HasCount(1, await bed.Library.Db.Entries.ListByAccountAsync(account.Id)); + } + + private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes)); + + private sealed class SyncProgress(Action report) : IProgress + { + public void Report(ImportProgress value) => report(value); + } +} diff --git a/tests/Febe.Tests/Import/MediaReaderTests.cs b/tests/Febe.Tests/Import/MediaReaderTests.cs index c660bde..2c64266 100644 --- a/tests/Febe.Tests/Import/MediaReaderTests.cs +++ b/tests/Febe.Tests/Import/MediaReaderTests.cs @@ -89,6 +89,35 @@ public sealed class MediaReaderTests Assert.IsNull(Mp4Reader.Read(new MemoryStream("not a movie at all"u8.ToArray()), Now)); } + /// + /// A 64-bit box size close to 2^63 overflows «offset + size»: the reader must stop on it, not walk backwards + /// forever (the analysis would hang and every «Riprendi» would hit the same file again). + /// + [TestMethod] + public void AHugeSixtyFourBitBoxSizeEndsTheReadingInsteadOfLooping() + { + static byte[] Box(string type, params byte[][] parts) => + [.. BigEndian((uint)(8 + parts.Sum(p => p.Length))), .. Encoding.ASCII.GetBytes(type), .. parts.SelectMany(p => p)]; + static byte[] BigEndian(uint value) => [(byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte)value]; + byte[] huge = [0, 0, 0, 1, .. "xxxx"u8, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF8]; + var ftyp = Box("ftyp", "isom"u8.ToArray(), [0, 0, 2, 0]); + + // Inside moov: a valid «free» box, then the damaged one. + byte[] insideMoov = [.. ftyp, .. Box("moov", Box("free"), huge)]; + + // At the top level, after ftyp. + byte[] topLevel = [.. ftyp, .. huge, .. new byte[32]]; + + foreach (var (name, bytes) in new[] { ("moov", insideMoov), ("primo livello", topLevel) }) + { + var reading = Task.Run(() => Mp4Reader.Read(new MemoryStream(bytes), Now)); + Assert.IsTrue(reading.Wait(TimeSpan.FromSeconds(10)), $"{name}: la lettura deve finire."); + var info = reading.Result; + Assert.IsNull(info?.CreatedAt, name); + Assert.IsNull(info?.Width, name); + } + } + [TestMethod] public void AnMp4MadeByFfmpegCarriesItsCreationTime() { diff --git a/tests/Febe.Tests/Library/DailyBackupTests.cs b/tests/Febe.Tests/Library/DailyBackupTests.cs index b818be1..e5e5508 100644 --- a/tests/Febe.Tests/Library/DailyBackupTests.cs +++ b/tests/Febe.Tests/Library/DailyBackupTests.cs @@ -52,9 +52,14 @@ public sealed class DailyBackupTests } await File.WriteAllTextAsync(Path.Combine(folder, "appunti.txt"), "non toccare"); - await File.WriteAllTextAsync(Path.Combine(folder, "febe-2026-10-13.sqlite.abcd1234.part"), "copia interrotta"); - var clock = new ManualClock(LibraryTimes.Noon(today)); + var interrupted = Path.Combine(folder, "febe-2026-10-12.sqlite.abcd1234.part"); + await File.WriteAllTextAsync(interrupted, "copia interrotta"); + File.SetLastWriteTimeUtc(interrupted, clock.Now.UtcDateTime.AddDays(-1)); + var writing = Path.Combine(folder, "febe-2026-10-13.sqlite.ef567890.part"); + await File.WriteAllTextAsync(writing, "copia in corso"); + File.SetLastWriteTimeUtc(writing, clock.Now.UtcDateTime.AddMinutes(-2)); + var backup = new DailyBackup(db.Maintenance, clock, folder); var result = await backup.RunAsync(); @@ -64,9 +69,81 @@ public sealed class DailyBackupTests Assert.AreEqual(today, kept[0]); Assert.AreEqual(today.AddDays(-6), kept[^1]); Assert.HasCount(5, result.Removed); // four old copies and the interrupted one + Assert.IsFalse(File.Exists(interrupted)); + Assert.IsTrue(File.Exists(writing), "A temporary copy written minutes ago may belong to a copy still running."); Assert.IsTrue(File.Exists(Path.Combine(folder, "appunti.txt"))); } + /// + /// «Fai una copia adesso» clicked twice, or clicked while the 3:20 copy runs: the copies take turns, and neither + /// one removes the temporary file of the other. + /// + [TestMethod] + public async Task TwoCopiesAtTheSameTimeBothSucceed() + { + using var temp = new TempFolder(); + var folder = temp.Combine("dati", "copie"); + var clock = new ManualClock(LibraryTimes.Noon(new DateOnly(2026, 10, 13))); + var database = new SlowBackups(); + var first = new DailyBackup(database, clock, folder).RunNowAsync(); + await database.FirstStarted.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + var second = new DailyBackup(database, clock, folder).RunNowAsync(); + await Task.WhenAny(second, Task.Delay(500)); // without turns, the second copy finishes here and its retention runs + database.ReleaseFirst.SetResult(); + + var results = await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10)); + Assert.IsTrue(results.All(r => r.Created is not null)); + Assert.AreEqual(1, database.MaxConcurrent, "One copy of a folder at a time."); + Assert.HasCount(1, Directory.GetFiles(folder, "*.sqlite")); + Assert.IsEmpty(Directory.GetFiles(folder, "*.part")); + } + + /// VACUUM INTO stand-in: the first copy waits until the test lets it finish. + private sealed class SlowBackups : Core.Storage.IDatabaseMaintenance + { + private int _calls; + private int _running; + private int _max; + + public TaskCompletionSource FirstStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseFirst { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int MaxConcurrent => Volatile.Read(ref _max); + + public string DatabasePath => "febe.sqlite"; + + public int SchemaVersion => 1; + + public async Task BackupToAsync(string destinationFile, CancellationToken ct = default) + { + var running = Interlocked.Increment(ref _running); + int seen; + while ((seen = Volatile.Read(ref _max)) < running && Interlocked.CompareExchange(ref _max, running, seen) != seen) + { + } + + try + { + await File.WriteAllTextAsync(destinationFile, "copia", ct); + if (Interlocked.Increment(ref _calls) == 1) + { + FirstStarted.SetResult(); + await ReleaseFirst.Task.WaitAsync(ct); + } + } + finally + { + Interlocked.Decrement(ref _running); + } + } + + public Task CheckIntegrityAsync(CancellationToken ct = default) => Task.FromResult(true); + + public Task OptimizeAsync(CancellationToken ct = default) => Task.CompletedTask; + } + [TestMethod] public void RecognizesOnlyItsOwnFileNames() { diff --git a/tests/Febe.Tests/Library/JournalBacklogTests.cs b/tests/Febe.Tests/Library/JournalBacklogTests.cs new file mode 100644 index 0000000..be0d3e3 --- /dev/null +++ b/tests/Febe.Tests/Library/JournalBacklogTests.cs @@ -0,0 +1,152 @@ +using Febe.Core.Abstractions; +using Febe.Core.Collection; +using Febe.Core.Collection.Downloads; +using Febe.Core.Domain; +using Febe.Core.Import; +using Febe.Core.Library; +using Febe.Core.Storage; +using Febe.Tests.Collection.Support; +using Febe.Tests.Import.Support; +using Febe.Tests.Support; + +namespace Febe.Tests.Library; + +/// +/// Every saved entry gets its _febe.jsonl line (spec §8.5), also when the journal cannot be written at that moment +/// (a sync client holding it, a network share that dropped) and when the last file of an entry to settle is not stored. +/// A file already in the library is never treated as «nothing written». +/// +[TestClass] +public sealed class JournalBacklogTests +{ + [TestMethod] + public async Task ADownloadedFileStaysInTheLibraryWhenTheJournalIsBusyAndItsLineFollows() + { + using var temp = new TempFolder("giornale"); + await using var lib = await TestLibrary.CreateAsync(temp); + var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001")); + await lib.SaveAsync(account, Make.Entry(account.Id, "prima", Make.T0.AddDays(-1)), [TestFile.Photo(1, 3)]); + var entryId = await AddAsync(lib.Db, account, "p1", Make.Photo() with { SourceUrl = "https://cdn.esempio/1.jpg" }); + var queue = Queue(lib, temp, _ => (FetchOutcome.Completed, TestFile.PhotoBytes(21, 64, 80))); + var journal = AccountJournal.PathFor(lib.Layout.AccountFolder(account)); + + DownloadReport report; + await using (new FileStream(journal, FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + { + report = await queue.RunOnceAsync(CancellationToken.None); + } + + Assert.AreEqual(1, report.Stored, report.ToString()); + var media = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(); + Assert.AreEqual(MediaState.Downloaded, media.State, "The file is in the library: it is not to download again."); + Assert.IsTrue(File.Exists(lib.Layout.FullPath(account, media.RelativePath!))); + Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.GetAsync(entryId))!.State, "It stays visible in the library."); + Assert.AreEqual(entryId.ToString(System.Globalization.CultureInfo.InvariantCulture), await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey)); + Assert.IsTrue((await lib.Db.Events.ListAfterAsync(0)).Any(e => e.Code == "giornale-in-sospeso")); + Assert.IsFalse((await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Any(l => l.Entry.Id == "p1")); + + // The next pass writes the line, even with nothing to download. + await queue.RunOnceAsync(CancellationToken.None); + var line = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Single(l => l.Entry.Id == "p1"); + Assert.AreEqual(media.RelativePath, line.Files!.Single().Path); + Assert.IsNull(await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey)); + + // So a rebuild after losing the database finds it, without downloading anything. + await lib.LoseDatabaseAsync(); + await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock).RebuildAsync(); + var again = await lib.Db.Accounts.FindByInstagramIdAsync("1001"); + Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.FindAsync(again!.Id, EntryKind.Post, "p1"))!.State); + } + + [TestMethod] + public async Task AnEntryGetsItsLineAlsoWhenItsLastFileIsSkippedOrFails() + { + using var temp = new TempFolder("giornale"); + await using var lib = await TestLibrary.CreateAsync(temp); + var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } }); + var tooLarge = await AddAsync(lib.Db, account, "p2", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p2.jpg" }, Make.Video(2) with { SourceUrl = "https://cdn.esempio/grande.mp4" }); + var refused = await AddAsync(lib.Db, account, "p3", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p3.jpg" }, Make.Photo(2) with { SourceUrl = "https://cdn.esempio/altrove.jpg" }); + var queue = Queue(lib, temp, uri => uri.AbsolutePath switch + { + "/grande.mp4" => (FetchOutcome.Completed, new byte[5000]), + "/altrove.jpg" => (FetchOutcome.HostNotAllowed, null), + "/p2.jpg" => (FetchOutcome.Completed, TestFile.PhotoBytes(31, 64, 80)), + _ => (FetchOutcome.Completed, TestFile.PhotoBytes(32, 64, 80)), + }); + + var report = await queue.RunOnceAsync(CancellationToken.None); + + Assert.AreEqual(2, report.Stored, report.ToString()); + Assert.AreEqual(1, report.Skipped); + Assert.AreEqual(1, report.Failed); + var lines = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines; + foreach (var (id, entryId) in new[] { ("p2", tooLarge), ("p3", refused) }) + { + var stored = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(m => m.Position == 1); + var line = lines.SingleOrDefault(l => l.Entry.Id == id); + Assert.IsNotNull(line, $"{id}: the entry is settled with a file in the library, so it has its line."); + Assert.AreEqual(stored.RelativePath, line.Files!.Single().Path); + } + } + + [TestMethod] + public async Task AnImportedFileStaysCopiedWhenTheJournalIsBusyAndItsLineFollows() + { + using var temp = new TempFolder("giornale"); + await using var bed = await ImportTestBed.CreateAsync(temp); + var (marta, _) = await bed.AccountWithPhotoAsync("marta.demo", "300", ImportTestFiles.Photo(30)); + bed.Write("marta.demo/nuova.jpg", ImportTestFiles.Photo(31)); + var run = await bed.AnalyzeAsync(); + var folder = bed.Library.Layout.AccountFolder(marta); + + await using (new FileStream(AccountJournal.PathFor(folder), FileMode.Open, FileAccess.ReadWrite, FileShare.None)) + { + await bed.Copier.CopyAsync(run.Id); + } + + var file = await bed.FileAsync(run.Id, "marta.demo/nuova.jpg"); + Assert.AreEqual(ImportFileState.Copied, file.State, file.LastError); + Assert.IsNotNull(file.EntryId); + Assert.AreEqual(MediaState.Imported, (await bed.Library.Db.Media.ListByEntryAsync(file.EntryId.Value)).Single().State); + Assert.IsFalse((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true)); + + // Any later copy (or download pass) writes the line that is still missing. + await bed.Copier.CopyAsync(run.Id); + Assert.IsTrue((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true)); + Assert.IsNull(await bed.Library.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey)); + Assert.AreEqual(ImportFileState.Copied, (await bed.FileAsync(run.Id, "marta.demo/nuova.jpg")).State); + } + + private static DownloadQueue Queue(TestLibrary lib, TempFolder temp, Func answer) => + new(lib.Db, lib.Ingest, new TableFetcher(answer), new FakeDataBudget(), lib.Clock, new SeededRandom(1), + new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 }); + + private static async Task AddAsync(IFebeStorage db, Account account, string id, params Media[] media) + { + var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1)); + var added = await db.Entries.AddAsync(entry, media.Select(m => m with { SourceSeenAt = Make.T0 }).ToList()); + return added.Stored.Entry.Id; + } + + /// A file fetcher answering from a table: no network at all. + private sealed class TableFetcher(Func answer) : IFileFetcher + { + public async Task FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct) + { + var (outcome, bytes) = answer(address); + if (bytes is null) + { + return new FetchResult(outcome, 0, null, "errore di prova"); + } + + if (options?.MaxBytes is { } max && bytes.Length > max) + { + await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct); + return new FetchResult(FetchOutcome.TooLarge, max, 200, "troppo grande"); + } + + await destination.WriteAsync(bytes, ct); + return new FetchResult(outcome, bytes.Length, 200, null); + } + } +} diff --git a/tests/Febe.Tests/Library/LibraryRebuildRecoveryTests.cs b/tests/Febe.Tests/Library/LibraryRebuildRecoveryTests.cs new file mode 100644 index 0000000..d8e827c --- /dev/null +++ b/tests/Febe.Tests/Library/LibraryRebuildRecoveryTests.cs @@ -0,0 +1,169 @@ +using Febe.Core.Domain; +using Febe.Core.Library; +using Febe.Tests.Support; + +namespace Febe.Tests.Library; + +/// +/// «Ricostruisci il database dalla libreria» (spec §8.5) on libraries that changed after their files were saved: files +/// removed or replaced by Alberto's choices, files that cannot be read. The rebuild respects the choices, never stops +/// on one file, and a second run adds nothing. +/// +[TestClass] +public sealed class LibraryRebuildRecoveryTests +{ + /// One file of a series removed by «Tieni quello in libreria» stays removed, not «file mancante». + [TestMethod] + public async Task AFileOfASeriesRemovedByADuplicateDecisionStaysRemoved() + { + using var temp = new TempFolder("ricostruzione"); + await using var lib = await TestLibrary.CreateAsync(temp); + var account = await lib.Db.Accounts.AddAsync(Make.Account("doppie.prova", "500")); + await lib.SaveAsync(account, Make.Entry(account.Id, "100", Make.T0.AddDays(-3)), [TestFile.Photo(1, 41, 1080, 1350)]); + var series = await lib.SaveAsync(account, Make.Entry(account.Id, "200", Make.T0), [TestFile.Photo(1, 41, 720, 900), TestFile.Photo(2, 77, 1080, 1080)]); + var review = new DownloadDuplicateReview(lib.Db, lib.Layout, lib.Journal, lib.Clock, lib.Thumbnails, lib.Index); + var pair = (await review.ListAsync()).Single(c => c.Candidate.EntryId == series.Entry.Id); + Assert.AreEqual(1, (await review.DecideAsync(pair.Pair.Id, DuplicateDecision.KeepExisting)).FilesRemoved); + var before = (await lib.Db.Entries.GetWithMediaAsync(series.Entry.Id))!; + + await lib.LoseDatabaseAsync(); + var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails); + var report = await rebuilder.RebuildAsync(); + + Assert.AreEqual(0, report.FilesMissing, string.Join("; ", report.Notes)); + var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("500"))!; + var entry = (await lib.Db.Entries.FindAsync(accountAgain.Id, EntryKind.Post, "200"))!; + var media = await lib.Db.Media.ListByEntryAsync(entry.Id); + Assert.AreEqual(MediaState.Deleted, media.Single(m => m.Position == 1).State, "Alberto's removal is respected: never looked for or downloaded again."); + Assert.AreEqual(MediaState.Downloaded, media.Single(m => m.Position == 2).State); + Assert.AreEqual(before.Media.Single(m => m.Position == 2).RelativePath, media.Single(m => m.Position == 2).RelativePath); + Assert.AreEqual(before.Entry.State, entry.State); + + var again = await rebuilder.RebuildAsync(); + Assert.AreEqual(0, again.EntriesCreated); + } + + /// + /// A journal written before replacements closed the old file: an imported entry without identifier appears twice, + /// once with the file that was replaced. Only the entry with its file comes back, on every run. + /// + [TestMethod] + public async Task AnEntryWithoutIdentifierWhoseFileWasReplacedComesBackOnce() + { + using var temp = new TempFolder("ricostruzione"); + await using var lib = await TestLibrary.CreateAsync(temp); + var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "600")); + var folder = lib.Layout.AccountFolder(account); + var bytes = TestFile.PhotoBytes(5, 800, 600); + Directory.CreateDirectory(Path.Combine(folder, "importati")); + await File.WriteAllBytesAsync(Path.Combine(folder, "importati", "IMG_1 (2).jpg"), bytes); + var entry = new Entry + { + AccountId = account.Id, + Kind = EntryKind.Post, + PublishedAt = Make.T0.AddDays(-400), + Provenance = Provenance.Imported, + DiscoveredAt = Make.T0, + }; + Media Imported(string path, string sha) => Make.Photo() with { State = MediaState.Imported, RelativePath = path, Sha256 = sha, Bytes = 100 }; + await lib.Journal.AppendSavedAsync(folder, account, entry, [Imported("importati/IMG_1.jpg", Make.Sha(1))]); + await lib.Journal.AppendSavedAsync(folder, account, entry, [Imported("importati/IMG_1 (2).jpg", Make.Sha(2))]); + + await lib.LoseDatabaseAsync(); + var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock); + var report = await rebuilder.RebuildAsync(); + + Assert.AreEqual(1, report.EntriesCreated); + Assert.AreEqual(1, report.FilesMissing, "The old file is reported..."); + Assert.IsTrue(report.Notes.Any(n => n.Contains("importati/IMG_1.jpg", StringComparison.Ordinal) && n.Contains("non ripristinato", StringComparison.Ordinal))); + var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("600"))!; + var entries = await lib.Db.Entries.ListByAccountAsync(accountAgain.Id); + Assert.HasCount(1, entries, "...but not restored as an entry of its own."); + Assert.AreEqual("importati/IMG_1 (2).jpg", (await lib.Db.Media.ListByEntryAsync(entries[0].Id)).Single().RelativePath); + + var again = await rebuilder.RebuildAsync(); + Assert.AreEqual(0, again.EntriesCreated, "A second run adds nothing."); + Assert.HasCount(1, await lib.Db.Entries.ListByAccountAsync(accountAgain.Id)); + } + + /// A remembered deletion of an imported entry without identifier is not added again by every run. + [TestMethod] + public async Task ARememberedDeletionWithoutIdentifierIsRestoredOnce() + { + using var temp = new TempFolder("ricostruzione"); + await using var lib = await TestLibrary.CreateAsync(temp); + var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "700")); + var imported = await lib.SaveAsync(account, Make.Entry(account.Id, "x", Make.T0.AddDays(-300)) with { InstagramId = null, Code = null, Provenance = Provenance.Imported }, [TestFile.Photo(1, 9)]); + await new LibraryDeletion(lib.Db, lib.Layout, lib.Journal, lib.Clock).DeleteAsync(imported.Entry.Id, DeletionMode.Remember); + + await lib.LoseDatabaseAsync(); + var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock); + Assert.AreEqual(1, (await rebuilder.RebuildAsync()).EntriesDeleted); + var again = await rebuilder.RebuildAsync(); + + Assert.AreEqual(0, again.EntriesCreated); + var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("700"))!; + var entries = await lib.Db.Entries.ListByAccountAsync(accountAgain.Id); + Assert.HasCount(1, entries); + Assert.AreEqual(EntryState.Deleted, entries[0].State); + Assert.AreEqual(imported.Media[0].Sha256, (await lib.Db.Media.ListByEntryAsync(entries[0].Id)).Single().Sha256, "The SHA-256 is remembered: an import does not bring it back."); + } + + /// + /// A file that cannot be read (here: held exclusively by another program) and an unreadable journal do not stop + /// the rebuild of everything else; the file stays linked as the journal recorded it. + /// + [TestMethod] + public async Task AnUnreadableFileOrJournalDoesNotStopTheRebuild() + { + using var temp = new TempFolder("ricostruzione"); + await using var lib = await TestLibrary.CreateAsync(temp); + var marta = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "800")); + var luca = await lib.Db.Accounts.AddAsync(Make.Account("luca.demo", "801")); + var zoe = await lib.Db.Accounts.AddAsync(Make.Account("zoe.demo", "802")); + var locked = await lib.SaveAsync(marta, Make.Entry(marta.Id, "a1", Make.T0.AddDays(-2)), [TestFile.Photo(1, 11)]); + var fine = await lib.SaveAsync(marta, Make.Entry(marta.Id, "a2", Make.T0.AddDays(-1)), [TestFile.Photo(1, 12)]); + await lib.SaveAsync(luca, Make.Entry(luca.Id, "b1", Make.T0), [TestFile.Photo(1, 13)]); + await lib.SaveAsync(zoe, Make.Entry(zoe.Id, "c1", Make.T0), [TestFile.Photo(1, 14)]); + await lib.LoseDatabaseAsync(); + + var lockedPath = lib.Layout.FullPath(marta, locked.Media[0].RelativePath!); + var report = await WhileHeldAsync( + [lockedPath, AccountJournal.PathFor(lib.Layout.AccountFolder(luca))], + () => new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails).RebuildAsync()); + + Assert.AreEqual(1, report.FilesUnreadable, string.Join("; ", report.Notes)); + Assert.AreEqual(2, report.FilesVerified, "The other files of the folder and of the following folders are verified."); + Assert.IsTrue(report.Notes.Any(n => n.Contains("luca.demo", StringComparison.Ordinal) && n.Contains("_febe.jsonl", StringComparison.Ordinal))); + var martaAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("800"))!; + var restored = (await lib.Db.Media.ListByEntryAsync((await lib.Db.Entries.FindAsync(martaAgain.Id, EntryKind.Post, "a1"))!.Id)).Single(); + Assert.AreEqual(MediaState.Downloaded, restored.State, "The file is where the journal says: linked again, not to be downloaded."); + Assert.AreEqual(locked.Media[0].RelativePath, restored.RelativePath); + Assert.AreEqual(locked.Media[0].Sha256, restored.Sha256); + StringAssert.Contains(restored.LastError, "illeggibile"); + Assert.IsNotNull(await lib.Db.Entries.FindAsync(martaAgain.Id, EntryKind.Post, "a2")); + Assert.IsNotNull(await lib.Db.Accounts.FindByInstagramIdAsync("802"), "The folders after the unreadable one are rebuilt too."); + Assert.IsNull(await lib.Db.Accounts.FindByInstagramIdAsync("801"), "The folder with the unreadable journal waits for the next run."); + + var later = await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails).RebuildAsync(); + Assert.AreEqual(1, later.EntriesCreated, "Once readable, the skipped folder is rebuilt."); + _ = fine; + } + + /// Runs while other handles hold the files with no sharing at all. + private static async Task WhileHeldAsync(IReadOnlyList paths, Func> action) + { + var handles = paths.Select(p => new FileStream(p, FileMode.Open, FileAccess.ReadWrite, FileShare.None)).ToList(); + try + { + return await action(); + } + finally + { + foreach (var handle in handles) + { + await handle.DisposeAsync(); + } + } + } +}