Importazione e dati: «Tieni il più grande» per gruppi, righe di _febe.jsonl mai perse, ricostruzione robusta (import-1, import-2, library-1, journal-1, rebuild-1, rebuild-2, rebuild-3, backup-1)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
This commit is contained in:
Claude committed 2026-10-14 00:53:10 +00:00
1 parent 78c92eb2a4
commit 628fe5db2b
13 files changed
+1145 -95

No files matched your search

@@ -127,6 +127,7 @@ public sealed class DownloadQueue
public async Task<DownloadReport> RunOnceAsync(CancellationToken ct)
{
Directory.CreateDirectory(_options.StagingFolder);
await _ingest.AppendPendingJournalAsync(ct).ConfigureAwait(false); // lines a busy _febe.jsonl refused earlier
var tally = new Tally();
var attempted = new ConcurrentDictionary<long, bool>();
var accounts = new ConcurrentDictionary<long, Account?>();
@@ -168,6 +169,15 @@ public sealed class DownloadQueue
}
private async Task ProcessAsync(PendingDownload pending, Tally tally, ConcurrentDictionary<long, Account?> accounts, ConcurrentDictionary<long, bool> singlePostQueued, CancellationToken ct)
{
await ProcessFileAsync(pending, tally, accounts, singlePostQueued, ct).ConfigureAwait(false);
// Whatever the outcome (stored, duplicate, skipped, failed), the entry may now be settled: its journal line is
// written once no file waits any more and at least one is in the library (spec §8.5).
await AppendJournalWhenSettledAsync(pending.Media.EntryId).ConfigureAwait(false);
}
private async Task ProcessFileAsync(PendingDownload pending, Tally tally, ConcurrentDictionary<long, Account?> accounts, ConcurrentDictionary<long, bool> singlePostQueued, CancellationToken ct)
{
var media = pending.Media;
if (!accounts.TryGetValue(pending.AccountId, out var account))
@@ -319,7 +329,6 @@ public sealed class DownloadQueue
{
tally.Add(ref tally.Stored);
Report(media, entry, DownloadStep.Stored);
await AppendJournalWhenCompleteAsync(entry, ct).ConfigureAwait(false);
}
else
{
@@ -393,12 +402,17 @@ public sealed class DownloadQueue
Report(media, entry, DownloadStep.Expired);
}
private async Task AppendJournalWhenCompleteAsync(Entry entry, CancellationToken ct)
/// <summary>
/// Appends the journal line of an entry once settled. A journal that cannot be written now (a sync client holding
/// <c>_febe.jsonl</c>, a share that dropped) never turns a stored file back into one to download: the line waits
/// in <see cref="LibraryIngest.TryAppendJournalAsync"/> and is written by a later pass.
/// </summary>
private async Task AppendJournalWhenSettledAsync(long entryId)
{
var files = await _storage.Media.ListByEntryAsync(entry.Id, ct).ConfigureAwait(false);
if (files.All(f => f.State is not (MediaState.ToDownload or MediaState.Expired)))
var files = await _storage.Media.ListByEntryAsync(entryId, CancellationToken.None).ConfigureAwait(false);
if (files.Any(f => EntryStateRules.IsPresent(f.State)) && files.All(f => f.State is not (MediaState.ToDownload or MediaState.Expired)))
{
await _ingest.AppendJournalAsync(entry.Id, ct).ConfigureAwait(false);
await _ingest.TryAppendJournalAsync(entryId, CancellationToken.None).ConfigureAwait(false);
}
}
+56 -10
View File
@@ -67,6 +67,7 @@ public sealed class ImportCopier(ImportEnvironment env)
var files = await LoadFilesAsync(runId, ct).ConfigureAwait(false);
var planner = await ImportPlanner.LoadAsync(env.Storage, runId, files, ct).ConfigureAwait(false);
var ingest = new LibraryIngest(env.Storage, env.Layout, env.Naming, env.Journal, env.Clock, env.Thumbnails, hashIndex: null, env.SetFileTimeToPublication);
await ingest.AppendPendingJournalAsync(ct).ConfigureAwait(false); // lines a busy _febe.jsonl refused earlier
var context = new CopyContext(run, layout, recognizers, ingest);
var total = files.Count;
var bytesTotal = files.Values.Sum(f => f.Bytes);
@@ -99,6 +100,7 @@ public sealed class ImportCopier(ImportEnvironment env)
progress?.Report(new ImportProgress(runId, ImportPhase.Copying, done, total, bytesDone, bytesTotal, file.RelativePath, new Dictionary<ImportOutcome, int>(), copied, env.Clock.Now));
}
await ingest.AppendPendingJournalAsync(CancellationToken.None).ConfigureAwait(false);
run = run with { State = ImportRunState.Completed, CompletedAt = env.Clock.Now, LastError = null };
await env.Storage.Imports.UpdateRunAsync(run, CancellationToken.None).ConfigureAwait(false);
await env.Storage.Events.AppendAsync(
@@ -226,15 +228,22 @@ public sealed class ImportCopier(ImportEnvironment env)
catch (ContentChangedException)
{
tally.Failed++;
return await SaveAsync(file with { State = ImportFileState.Failed, LastError = ImportText.NoteChanged }).ConfigureAwait(false);
return await SaveAsync(await CurrentAsync(file).ConfigureAwait(false) with { State = ImportFileState.Failed, LastError = ImportText.NoteChanged }).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
tally.Failed++;
return await SaveAsync(file with { State = ImportFileState.Failed, LastError = ImportAnalyzer.ErrorText(ex) }).ConfigureAwait(false);
return await SaveAsync(await CurrentAsync(file).ConfigureAwait(false) with { State = ImportFileState.Failed, LastError = ImportAnalyzer.ErrorText(ex) }).ConfigureAwait(false);
}
}
/// <summary>
/// The file's record as saved during the attempt (the entry it was given, so a resumed copy reuses it instead of
/// taking the file for one already in the library).
/// </summary>
private async Task<ImportFile> CurrentAsync(ImportFile file) =>
await env.Storage.Imports.GetFileAsync(file.Id, CancellationToken.None).ConfigureAwait(false) ?? file;
/// <summary>Copies one file into its destination account as an imported entry (or into an entry it completes).</summary>
private async Task<ImportFile> ImportAsync(CopyContext context, ImportFile file, Tally tally, CancellationToken ct)
{
@@ -267,7 +276,7 @@ public sealed class ImportCopier(ImportEnvironment env)
if (EntryStateRules.IsPresent(media.State) && string.Equals(media.Sha256, file.Sha256, StringComparison.Ordinal) && media.RelativePath is { } already)
{
// Stored before an interruption: only the journal line and the state are missing.
await context.Ingest.AppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
await context.Ingest.TryAppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
return await FinishAsync(context, file, account, entry.Id, already, null, tally).ConfigureAwait(false);
}
@@ -329,7 +338,7 @@ public sealed class ImportCopier(ImportEnvironment env)
await env.Storage.Highlights.AddItemAsync(collection.Id, entry.Id, none).ConfigureAwait(false);
}
await context.Ingest.AppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
await context.Ingest.TryAppendJournalAsync(entry.Id, CancellationToken.None).ConfigureAwait(false);
return await FinishAsync(context, file, account, entry.Id, result.File.RelativePath, null, tally).ConfigureAwait(false);
}
@@ -444,6 +453,14 @@ public sealed class ImportCopier(ImportEnvironment env)
return await FinishAsync(context, file, (await AccountOfEntryAsync(media.EntryId, ct).ConfigureAwait(false)).Account, media.EntryId, media.RelativePath, ImportText.NoteReplaced, tally).ConfigureAwait(false);
}
// Alberto chose «keep the larger»: the file in the library now is what counts, not the one seen when the copy
// was planned. A smaller or equal file never takes the place of a larger one.
if (!ImportPlanner.IsLarger(file.Width, file.Height, file.Bytes, media.Width, media.Height, media.Bytes))
{
tally.Skipped++;
return await SaveAsync(file with { State = ImportFileState.Skipped, MatchedMediaId = media.Id, LastError = ImportText.NoteLibraryLarger }).ConfigureAwait(false);
}
var (account, entry) = await AccountOfEntryAsync(media.EntryId, ct).ConfigureAwait(false);
var folder = env.Layout.AccountFolder(account);
var source = LibraryLayout.FullPath(context.Layout.Root, file.RelativePath);
@@ -473,9 +490,19 @@ public sealed class ImportCopier(ImportEnvironment env)
ct).ConfigureAwait(false))!;
}
// From here the replacement goes to the end: a half-done replacement is worse than a late one.
// From here the replacement goes to the end: a half-done replacement is worse than a late one. The new file is
// either registered or removed again: never left in the library with nothing pointing to it.
var none = CancellationToken.None;
var onDisk = await env.FileSystem.ComputeSha256Async(saved.FullPath, none).ConfigureAwait(false);
string? onDisk = null;
try
{
onDisk = await env.FileSystem.ComputeSha256Async(saved.FullPath, none).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Not readable: not verified.
}
if (!string.Equals(onDisk, file.Sha256, StringComparison.Ordinal))
{
TryDelete(saved.FullPath); // the new file was never registered: nothing in the library points to it
@@ -488,14 +515,19 @@ public sealed class ImportCopier(ImportEnvironment env)
var thumbnail = false;
if (file.MediaKind == MediaKind.Photo)
{
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
await using (var stream = File.OpenRead(saved.FullPath))
try
{
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
await using var stream = File.OpenRead(saved.FullPath);
if (ImageDecoding.ReadSize(stream) is { } size)
{
(width, height) = size;
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Verified a moment ago: recorded without perceptual hash, with the size measured by the analysis.
}
thumbnail = TryThumbnail(saved.Sha256, () => File.OpenRead(saved.FullPath));
}
@@ -513,7 +545,21 @@ public sealed class ImportCopier(ImportEnvironment env)
media.State,
env.Clock.Now,
none).ConfigureAwait(false);
await context.Ingest.AppendJournalAsync(entry.Id, none).ConfigureAwait(false);
await context.Ingest.TryAppendJournalAsync(entry.Id, none).ConfigureAwait(false);
if (entry.InstagramId is null && string.IsNullOrWhiteSpace(entry.Code) && !string.Equals(oldRelative, saved.RelativePath, StringComparison.Ordinal))
{
// An entry without identifier or code is known in the journal by its file: the line of the old file is
// closed («delete», not remembered), so a rebuild does not bring back an entry whose file is gone.
try
{
await env.Journal.AppendDeletedAsync(folder, account, entry, [oldRelative], remember: false, none).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Without this line a rebuild reports the old file as missing and does not restore it: nothing is lost.
}
}
if (hash is { } h)
{
env.HashIndex?.Add(media.Id, account.Id, h);
@@ -538,7 +584,7 @@ public sealed class ImportCopier(ImportEnvironment env)
{
thumbnails.Delete(oldSha);
}
catch (IOException)
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// A thumbnail left behind costs a few kilobytes.
}
+48 -15
View File
@@ -41,8 +41,10 @@ internal sealed record PlannedFile(ImportFile File, PlannedAction Action, string
/// source;</item>
/// <item>«Tieni quello in libreria»: the source file is skipped. «Tienili entrambi»: it is imported. «Tieni il più grande»:
/// if the library file is larger the source file is skipped, otherwise it replaces the library file (more pixels win,
/// then more bytes; a tie keeps the library file). Inside the source, among files joined by «tieni il più grande» only
/// the largest is imported;</item>
/// then more bytes; a tie keeps the library file). Several source files paired with the same library file form one
/// group: only the largest of them can replace it, the others are skipped, so a replacement never removes the copy
/// another replacement has just put in the library. Inside the source, among files joined by «tieni il più grande»
/// only the largest is imported;</item>
/// <item>files of a folder without account stay out.</item>
/// </list>
/// </summary>
@@ -88,6 +90,7 @@ internal sealed class ImportPlanner
}
var groups = new UnionFind();
var rivals = new Dictionary<long, List<long>>(); // library file → source files that may replace it
foreach (var pair in pairs.Where(p => p.Decision != DuplicateDecision.Pending && p.CandidateImportFileId is not null))
{
var candidate = pair.CandidateImportFileId!.Value;
@@ -99,17 +102,13 @@ internal sealed class ImportPlanner
planner.Lose(candidate, ImportText.NoteKeptExisting);
break;
case DuplicateDecision.KeepLarger:
var library = await storage.Media.GetAsync(mediaId, ct).ConfigureAwait(false);
if (library is not null && files.TryGetValue(candidate, out var file)
&& IsLarger(file.Width, file.Height, file.Bytes, library.Width, library.Height, library.Bytes))
if (!rivals.TryGetValue(mediaId, out var list))
{
planner._replacements[candidate] = mediaId;
}
else
{
planner.Lose(candidate, ImportText.NoteLibraryLarger);
list = [];
rivals[mediaId] = list;
}
list.Add(candidate);
break;
default:
break; // KeepBoth: imported
@@ -134,20 +133,54 @@ internal sealed class ImportPlanner
foreach (var group in groups.Groups())
{
var members = group.Where(files.ContainsKey).Select(id => files[id]).ToList();
var winner = members
.OrderByDescending(f => (long)(f.Width ?? 0) * (f.Height ?? 0))
.ThenByDescending(f => f.Bytes)
.ThenBy(f => f.Id)
.FirstOrDefault();
var winner = Largest(members);
foreach (var member in members.Where(m => m.Id != winner?.Id))
{
planner.Lose(member.Id, ImportText.NoteKeptOtherInSource);
}
}
// «Tieni il più grande» against a library file: the library file and every source file paired with it are one
// group, and only the largest of the group stays. Deciding pair by pair would let a second, smaller source
// file replace the copy the first one has just put in the library (and, with «Sposta», lose the larger one).
foreach (var (mediaId, candidates) in rivals)
{
var library = await storage.Media.GetAsync(mediaId, ct).ConfigureAwait(false);
// A file already skipped never entered the library: it does not compete.
var members = candidates.Distinct()
.Where(id => files.TryGetValue(id, out var f) && f.State != ImportFileState.Skipped && !planner._decided.ContainsKey(id))
.Select(id => files[id])
.ToList();
var winner = Largest(members);
var replaces = winner is not null && library is not null
&& (IsLarger(winner.Width, winner.Height, winner.Bytes, library.Width, library.Height, library.Bytes)
// The library already holds this very file: a replacement interrupted before its state was saved,
// which the copy finishes.
|| string.Equals(winner.Sha256, library.Sha256, StringComparison.Ordinal));
foreach (var member in members)
{
if (replaces && member.Id == winner!.Id)
{
planner._replacements[member.Id] = mediaId;
}
else
{
planner.Lose(member.Id, replaces ? ImportText.NoteLargerInSource : ImportText.NoteLibraryLarger);
}
}
}
return planner;
}
/// <summary>The largest file: more pixels, then more bytes; on a tie the first one analysed.</summary>
private static ImportFile? Largest(IEnumerable<ImportFile> files) =>
files
.OrderByDescending(f => (long)(f.Width ?? 0) * (f.Height ?? 0))
.ThenByDescending(f => f.Bytes)
.ThenBy(f => f.Id)
.FirstOrDefault();
/// <summary>True when the folder of the file has (or will have) an account.</summary>
public bool HasTarget(ImportFile file) => file.TargetAccountId is not null || _newAccounts.ContainsKey(file.Folder);
+1
View File
@@ -136,6 +136,7 @@ public static class ImportText
internal const string NoteKeptExisting = "Hai scelto di tenere il file già in libreria.";
internal const string NoteLibraryLarger = "Hai scelto di tenere il più grande: è quello già in libreria.";
internal const string NoteKeptOtherInSource = "Hai scelto di tenere l'altro file della sorgente.";
internal const string NoteLargerInSource = "Hai scelto di tenere il più grande: è un altro file della sorgente, che prende il posto di quello in libreria.";
internal const string NoteNotRecognized = "Né foto né video: ignorato.";
internal const string NoteAlreadyInLibrary = "Identico a un file già in libreria: saltato.";
internal const string NoteReplaced = "Ha sostituito il file più piccolo della libreria, come hai scelto.";
+9 -3
View File
@@ -25,6 +25,9 @@ public static class Mp4Reader
private const int MaxTopLevelBoxes = 10_000;
/// <summary>The most child boxes read inside one box (real files have a few dozen).</summary>
private const int MaxChildBoxes = 10_000;
/// <summary>Reads a file (read-only). Null when no <c>moov</c> box is found.</summary>
/// <param name="path">The file.</param>
/// <param name="now">Current time: creation times after it (by more than a day) are ignored.</param>
@@ -72,7 +75,8 @@ public static class Mp4Reader
size = length - position; // to the end of the file
}
if (size < headerSize || position + size > length + 8)
// Compared without adding to the position: a 64-bit size close to 2^63 would overflow the sum and pass.
if (size < headerSize || size > length - position + 8)
{
return null; // damaged
}
@@ -193,7 +197,7 @@ public static class Mp4Reader
{
var list = new List<(string, int, int)>();
var offset = 0;
while (offset + 8 <= data.Length)
while (offset + 8 <= data.Length && list.Count < MaxChildBoxes)
{
long size = BinaryPrimitives.ReadUInt32BigEndian(data.Slice(offset, 4));
var type = System.Text.Encoding.Latin1.GetString(data.Slice(offset + 4, 4));
@@ -208,7 +212,9 @@ public static class Mp4Reader
size = data.Length - offset;
}
if (size < headerSize || offset + size > data.Length)
// «size > remaining» cannot overflow (unlike «offset + size»), and with «size >= headerSize» it keeps the
// offset moving forward: a damaged 64-bit size can neither loop back nor run past the end.
if (size < headerSize || size > data.Length - offset)
{
break;
}
+99 -37
View File
@@ -1,3 +1,4 @@
using System.Collections.Concurrent;
using System.Globalization;
using System.Text.RegularExpressions;
using Febe.Core.Abstractions;
@@ -18,6 +19,11 @@ public sealed record BackupResult(BackupFile? Created, IReadOnlyList<string> Rem
/// Daily copy of the database (spec §8.5): <c>VACUUM INTO</c> a new file <c>Dati/copie/febe-YYYY-MM-DD.sqlite</c>
/// (local date), keeping seven days. A copy is written under a temporary name and renamed, so a cut copy never has a
/// final name. Only files named <c>febe-YYYY-MM-DD.sqlite</c> are ever removed.
/// <para>
/// The 3:20 loop and «Fai una copia adesso» each make their own instance: copies and retention of one folder take turns
/// (one gate per folder), and the retention removes only temporary copies left behind long ago, never one still being
/// written.
/// </para>
/// </summary>
public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock clock, string backupsFolder, int keepDays = DailyBackup.DefaultKeepDays)
{
@@ -27,6 +33,12 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
private const string Extension = ".sqlite";
private const string PartSuffix = ".part";
/// <summary>A temporary copy untouched for this long was left by a copy that stopped (crash, power cut).</summary>
private static readonly TimeSpan StalePart = TimeSpan.FromHours(6);
private static readonly ConcurrentDictionary<string, SemaphoreSlim> Gates = new(
OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal);
[GeneratedRegex(@"^febe-(\d{4}-\d{2}-\d{2})\.sqlite$", RegexOptions.CultureInvariant)]
private static partial Regex BackupName();
@@ -36,32 +48,39 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
/// <summary>File name of the copy of a day.</summary>
public static string FileNameFor(DateOnly day) => Prefix + day.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) + Extension;
private SemaphoreSlim Gate => Gates.GetOrAdd(Folder, _ => new SemaphoreSlim(1, 1));
/// <summary>Makes today's copy if missing, then applies the retention. Safe to call often.</summary>
public async Task<BackupResult> RunAsync(CancellationToken ct = default)
{
Directory.CreateDirectory(Folder);
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
var target = Path.Combine(Folder, FileNameFor(today));
BackupFile? created = null;
if (!File.Exists(target))
await Gate.WaitAsync(ct).ConfigureAwait(false);
try
{
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
try
Directory.CreateDirectory(Folder);
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
var target = Path.Combine(Folder, FileNameFor(today));
BackupFile? created = null;
if (!File.Exists(target))
{
await database.BackupToAsync(part, ct).ConfigureAwait(false);
File.Move(part, target, overwrite: false);
created = new BackupFile(today, target, new FileInfo(target).Length);
}
finally
{
if (File.Exists(part))
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
try
{
File.Delete(part);
await database.BackupToAsync(part, ct).ConfigureAwait(false);
File.Move(part, target, overwrite: false);
created = new BackupFile(today, target, new FileInfo(target).Length);
}
finally
{
TryDelete(part);
}
}
}
return new BackupResult(created, ApplyRetention(today));
return new BackupResult(created, Retain(today));
}
finally
{
Gate.Release();
}
}
/// <summary>
@@ -71,26 +90,31 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
/// </summary>
public async Task<BackupResult> RunNowAsync(CancellationToken ct = default)
{
Directory.CreateDirectory(Folder);
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
var target = Path.Combine(Folder, FileNameFor(today));
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
BackupFile created;
await Gate.WaitAsync(ct).ConfigureAwait(false);
try
{
await database.BackupToAsync(part, ct).ConfigureAwait(false);
File.Move(part, target, overwrite: true);
created = new BackupFile(today, target, new FileInfo(target).Length);
Directory.CreateDirectory(Folder);
var today = LocalCalendar.Today(clock.Now, clock.LocalZone);
var target = Path.Combine(Folder, FileNameFor(today));
var part = target + "." + Guid.NewGuid().ToString("N")[..8] + PartSuffix;
BackupFile created;
try
{
await database.BackupToAsync(part, ct).ConfigureAwait(false);
File.Move(part, target, overwrite: true);
created = new BackupFile(today, target, new FileInfo(target).Length);
}
finally
{
TryDelete(part);
}
return new BackupResult(created, Retain(today));
}
finally
{
if (File.Exists(part))
{
File.Delete(part);
}
Gate.Release();
}
return new BackupResult(created, ApplyRetention(today));
}
/// <summary>The copies present, newest first.</summary>
@@ -114,10 +138,25 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
}
/// <summary>
/// Removes copies older than <see cref="DefaultKeepDays"/> days (today included) and stale temporary copies.
/// Returns the removed paths.
/// Removes copies older than <see cref="DefaultKeepDays"/> days (today included) and temporary copies left behind
/// by a copy that stopped (untouched for hours: one still being written is never removed). A file that cannot be
/// removed now (open in another program) is left for the next time. Returns the removed paths.
/// </summary>
public IReadOnlyList<string> ApplyRetention(DateOnly today)
{
Gate.Wait();
try
{
return Retain(today);
}
finally
{
Gate.Release();
}
}
/// <summary>The retention; the caller holds the gate.</summary>
private List<string> Retain(DateOnly today)
{
var removed = new List<string>();
if (!Directory.Exists(Folder))
@@ -128,19 +167,42 @@ public sealed partial class DailyBackup(IDatabaseMaintenance database, IClock cl
var oldestKept = today.AddDays(-(Math.Max(1, keepDays) - 1));
foreach (var backup in List().Where(b => b.Day < oldestKept))
{
File.Delete(backup.Path);
removed.Add(backup.Path);
if (TryDelete(backup.Path))
{
removed.Add(backup.Path);
}
}
var staleBefore = clock.Now.UtcDateTime - StalePart;
foreach (var part in Directory.EnumerateFiles(Folder, Prefix + "*" + PartSuffix))
{
File.Delete(part);
removed.Add(part);
if (File.GetLastWriteTimeUtc(part) < staleBefore && TryDelete(part))
{
removed.Add(part);
}
}
return removed;
}
private static bool TryDelete(string path)
{
try
{
if (!File.Exists(path))
{
return false;
}
File.Delete(path);
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return false; // the next retention tries again
}
}
/// <summary>The day of a backup file name, or null for any other file.</summary>
public static DateOnly? TryParseDay(string fileName)
{
+142 -2
View File
@@ -1,3 +1,4 @@
using System.Globalization;
using Febe.Core.Abstractions;
using Febe.Core.Domain;
using Febe.Core.Storage;
@@ -87,6 +88,14 @@ public sealed class LibraryIngest(
bool setFileTimeToPublication = true,
SafeFileWriter? writer = null)
{
/// <summary>
/// Setting with the entries whose <c>_febe.jsonl</c> line is still to write (ids, comma separated): it survives a
/// restart, and downloads and imports share it.
/// </summary>
internal const string PendingJournalKey = "libreria.giornale-da-scrivere";
private static readonly SemaphoreSlim PendingJournalGate = new(1, 1);
private readonly SafeFileWriter _writer = writer ?? new SafeFileWriter(layout.MaxPathLength);
private volatile LibraryNaming _naming = naming;
private volatile bool _setFileTimeToPublication = setFileTimeToPublication;
@@ -191,19 +200,27 @@ public sealed class LibraryIngest(
return new MediaSaveResult(MediaSaveOutcome.IdenticalInAccount, null, identical, []);
}
// From here the file is in the library under its final name: it is always recorded, even when reading it again
// for the perceptual hash and the size fails (callers treat an IOException as «nothing written»).
ulong? hash = null;
int? width = media.Width, height = media.Height;
var hasThumbnail = false;
if (media.Kind == MediaKind.Photo)
{
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
await using (var stream = File.OpenRead(saved.FullPath))
try
{
hash = PerceptualHash.ComputeFromFile(saved.FullPath);
await using var stream = File.OpenRead(saved.FullPath);
if (ImageDecoding.ReadSize(stream) is { } size)
{
(width, height) = size;
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Recorded without perceptual hash (no duplicate check by look for this file) and with the size the
// page gave; a rebuild of the database computes both again.
}
hasThumbnail = TryThumbnail(saved.Sha256, () => File.OpenRead(saved.FullPath), owned: true);
}
@@ -261,6 +278,129 @@ public sealed class LibraryIngest(
ct).ConfigureAwait(false);
}
/// <summary>
/// Appends the <c>_febe.jsonl</c> line of an entry whose files are already in the library. When the journal cannot
/// be written now (held by a sync client, a network share that dropped, a full disk), the files stay recorded as
/// they are and the entry is remembered: <see cref="AppendPendingJournalAsync"/> writes its line later, so a rebuild
/// of the database still finds it (spec §8.5). Returns false in that case.
/// </summary>
public async Task<bool> TryAppendJournalAsync(long entryId, CancellationToken ct = default)
{
try
{
await AppendJournalAsync(entryId, ct).ConfigureAwait(false);
return true;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
await RememberPendingJournalAsync(entryId).ConfigureAwait(false);
return false;
}
}
/// <summary>
/// Writes the journal lines that could not be written before (see <see cref="TryAppendJournalAsync"/>), each with the
/// entry as it is now. Entries deleted meanwhile, or left without files, need no line. Returns how many still wait.
/// </summary>
public async Task<int> AppendPendingJournalAsync(CancellationToken ct = default)
{
await PendingJournalGate.WaitAsync(ct).ConfigureAwait(false);
try
{
var pending = await ReadPendingJournalAsync().ConfigureAwait(false);
if (pending.Count == 0)
{
return 0;
}
var left = new List<long>();
foreach (var entryId in pending)
{
var stored = await storage.Entries.GetWithMediaAsync(entryId, CancellationToken.None).ConfigureAwait(false);
if (stored is null || stored.Entry.State == EntryState.Deleted || !stored.Media.Any(m => EntryStateRules.IsPresent(m.State)))
{
continue;
}
if (ct.IsCancellationRequested)
{
left.Add(entryId);
continue;
}
try
{
await AppendJournalAsync(entryId, CancellationToken.None).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
left.Add(entryId);
}
}
await WritePendingJournalAsync(left).ConfigureAwait(false);
return left.Count;
}
finally
{
PendingJournalGate.Release();
}
}
private async Task RememberPendingJournalAsync(long entryId)
{
await PendingJournalGate.WaitAsync(CancellationToken.None).ConfigureAwait(false);
try
{
var pending = await ReadPendingJournalAsync().ConfigureAwait(false);
if (pending.Contains(entryId))
{
return;
}
await WritePendingJournalAsync([.. pending, entryId]).ConfigureAwait(false);
if (pending.Count == 0)
{
var accountId = (await storage.Entries.GetAsync(entryId, CancellationToken.None).ConfigureAwait(false))?.AccountId;
await storage.Events.AppendAsync(
new EventRecord
{
At = clock.Now,
Code = "giornale-in-sospeso",
Severity = EventSeverity.Warning,
Message = "Non si è potuto scrivere _febe.jsonl nella cartella dell'account (file in uso da un altro programma o disco non raggiungibile). I file sono al sicuro in libreria: Febe scrive le righe mancanti appena può.",
AccountId = accountId,
},
CancellationToken.None).ConfigureAwait(false);
}
}
finally
{
PendingJournalGate.Release();
}
}
private async Task<List<long>> ReadPendingJournalAsync()
{
var text = await storage.Settings.GetAsync(PendingJournalKey, CancellationToken.None).ConfigureAwait(false);
var ids = new List<long>();
foreach (var part in (text ?? string.Empty).Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
if (long.TryParse(part, NumberStyles.Integer, CultureInfo.InvariantCulture, out var id) && !ids.Contains(id))
{
ids.Add(id);
}
}
return ids;
}
private Task WritePendingJournalAsync(IReadOnlyList<long> ids) =>
storage.Settings.SetAsync(
PendingJournalKey,
ids.Count == 0 ? null : string.Join(',', ids.Select(id => id.ToString(CultureInfo.InvariantCulture))),
CancellationToken.None);
private bool TryThumbnail(string key, Func<Stream> open, bool owned)
{
if (thumbnails is null)
+167 -21
View File
@@ -48,6 +48,12 @@ public sealed record RebuildReport
public int FilesMissing { get; init; }
/// <summary>
/// Files that are on disk but could not be read (locked by another program, no permission, damaged disk): kept as
/// the journal describes them, without verification, and listed in <see cref="Notes"/>.
/// </summary>
public int FilesUnreadable { get; init; }
/// <summary>Files whose SHA-256 differs from the journal (the file on disk wins).</summary>
public int HashMismatches { get; init; }
@@ -113,7 +119,7 @@ public sealed class LibraryRebuilder(
{
ct.ThrowIfCancellationRequested();
await RebuildFolderAsync(folders[i], options, report, ct).ConfigureAwait(false);
progress?.Report(new RebuildProgress(folders[i], i + 1, folders.Count, report.FilesVerified + report.FilesMissing));
progress?.Report(new RebuildProgress(folders[i], i + 1, folders.Count, report.FilesVerified + report.FilesMissing + report.FilesUnreadable));
}
report.AccountFolders = folders.Count;
@@ -122,7 +128,18 @@ public sealed class LibraryRebuilder(
private async Task RebuildFolderAsync(string folder, RebuildOptions options, Accumulator report, CancellationToken ct)
{
var journal = await AccountJournal.ReadAsync(folder, ct).ConfigureAwait(false);
JournalContent journal;
try
{
journal = await AccountJournal.ReadAsync(folder, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// One unreadable journal does not stop the others: this folder can be rebuilt again later.
report.Note($"«{Path.GetFileName(folder)}»: _febe.jsonl non si può leggere ({Reason(ex)}); la cartella non è stata ricostruita.");
return;
}
report.InvalidLines += journal.InvalidLines;
if (journal.Lines.Count == 0)
{
@@ -157,8 +174,26 @@ public sealed class LibraryRebuilder(
foreach (var item in items)
{
ct.ThrowIfCancellationRequested();
if (!item.Deleted && !item.HasIdentity && item.Files.Count > 0 && item.Files.Keys.All(n => checks[(item, n)].IsMissing))
{
// An entry without identifier or code whose files are all gone has nothing to show and nothing to download
// again: it is reported, not restored. (A journal written before file replacements recorded the old path
// as deleted holds such an entry for every imported file replaced by «Tieni il più grande».)
foreach (var file in item.Files.Values)
{
report.FilesMissing++;
report.Note($"File mancante: {file.Path} (elemento senza identificativo, non ripristinato)");
}
continue;
}
var entry = BuildEntry(account.Id, item);
var media = item.Files.Values.OrderBy(f => f.N).Select(f => BuildMedia(item, f, item.Deleted ? null : checks[(item, f.N)], report)).ToList();
var media = item.Files.Values
.Select(f => BuildMedia(item, f, item.Deleted ? null : checks[(item, f.N)], report))
.Concat(item.Removed.Values.Select(BuildRemoved))
.OrderBy(m => m.Position)
.ToList();
batch.Add((item, new NewEntry(entry, media)));
if (batch.Count >= BatchSize)
{
@@ -192,7 +227,8 @@ public sealed class LibraryRebuilder(
{
var present = item.InstagramId is { } id
? known.Contains((item.Kind, id))
: item.Files.Values.Any(f => paths.ContainsKey(LibraryLayout.ToStored(f.Path)));
: item.Files.Values.Any(f => paths.ContainsKey(LibraryLayout.ToStored(f.Path)))
|| (item.Deleted && await IsDeletionKnownAsync(account, item, ct).ConfigureAwait(false));
if (!present)
{
missing.Add(item);
@@ -216,6 +252,24 @@ public sealed class LibraryRebuilder(
return missing;
}
/// <summary>
/// A remembered deletion of an entry without identifier has no file path left in the database: it is recognised by
/// the SHA-256 of its files, so a second rebuild does not add it again.
/// </summary>
private async Task<bool> IsDeletionKnownAsync(Account account, Item item, CancellationToken ct)
{
foreach (var sha in item.Files.Values.Select(f => f.Sha256).OfType<string>().Where(s => s.Length == 64))
{
var same = await storage.Media.FindBySha256Async(sha.ToLowerInvariant(), ct).ConfigureAwait(false);
if (same.Any(s => s.Entry.AccountId == account.Id && s.Entry.Kind == item.Kind && s.Entry.State == EntryState.Deleted))
{
return true;
}
}
return false;
}
private async Task<Account> ResolveAccountAsync(string folder, IReadOnlyList<JournalLine> lines, Accumulator report, CancellationToken ct)
{
var last = lines[^1].Account;
@@ -255,7 +309,11 @@ public sealed class LibraryRebuilder(
return await storage.Accounts.AddAsync(created, ct).ConfigureAwait(false);
}
/// <summary>Replays the journal: the last save of an entry wins, deletions remove it or mark it.</summary>
/// <summary>
/// Replays the journal: the last save of an entry wins, deletions remove it or mark it. Every save line lists all
/// the files present at that moment, so a position a later save no longer lists was removed (one file of a series
/// dropped by a duplicate decision): it comes back as «eliminato», never as a file to find or download again.
/// </summary>
private static List<Item> Fold(IReadOnlyList<JournalLine> lines, Accumulator report)
{
var order = new List<Item>();
@@ -286,11 +344,20 @@ public sealed class LibraryRebuilder(
else
{
item.Last = line;
if (line.Files is { Count: > 0 } listed)
{
foreach (var dropped in item.Files.Keys.Where(n => !listed.Any(f => f.N == n)).ToList())
{
item.Removed[dropped] = item.Files[dropped];
item.Files.Remove(dropped);
}
}
}
foreach (var file in line.Files ?? [])
{
item.Files[file.N] = file;
item.Removed.Remove(file.N);
}
}
else if (line.Remember == true)
@@ -315,6 +382,10 @@ public sealed class LibraryRebuilder(
return order;
}
/// <summary>
/// The identity of an entry across its lines: the Instagram id; else the code (an imported entry known by its code
/// keeps it when one of its files is replaced); else the first file, for entries without either.
/// </summary>
private static string KeyOf(EntryKind kind, JournalLine line)
{
if (!string.IsNullOrWhiteSpace(line.Entry.Id))
@@ -322,6 +393,11 @@ public sealed class LibraryRebuilder(
return $"{kind}|id|{line.Entry.Id.Trim()}";
}
if (!string.IsNullOrWhiteSpace(line.Entry.Code))
{
return $"{kind}|code|{line.Entry.Code.Trim()}";
}
var firstPath = line.Files?.OrderBy(f => f.N).FirstOrDefault()?.Path;
return firstPath is not null
? $"{kind}|path|{firstPath}"
@@ -345,47 +421,71 @@ public sealed class LibraryRebuilder(
return FileCheck.Missing;
}
// A file that cannot be read (locked by another program, no permission, bad sectors, a cloud placeholder that
// cannot be fetched offline) must not stop the rebuild of everything else.
string sha;
long bytes;
await using (var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 128 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan))
try
{
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 128 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan);
bytes = stream.Length;
sha = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, ct).ConfigureAwait(false));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
var recorded = file.Sha256 is { Length: 64 } s ? s.ToLowerInvariant() : null;
return FileCheck.CannotRead(Reason(ex)) with { HasThumbnail = recorded is not null && (thumbnails?.Exists(recorded) ?? false) };
}
var isVideo = string.Equals(file.Type, "video", StringComparison.OrdinalIgnoreCase);
ulong? phash = null;
int? width = file.Width, height = file.Height;
var thumbnail = thumbnails?.Exists(sha) ?? false;
if (!isVideo)
try
{
phash = PerceptualHash.ComputeFromFile(path);
await using (var stream = File.OpenRead(path))
if (!isVideo)
{
if (ImageDecoding.ReadSize(stream) is { } size)
phash = PerceptualHash.ComputeFromFile(path);
await using (var stream = File.OpenRead(path))
{
(width, height) = size;
if (ImageDecoding.ReadSize(stream) is { } size)
{
(width, height) = size;
}
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Hashed but not decoded a second time: the file stays, without perceptual hash.
}
if (createThumbnails && thumbnails is not null && !thumbnail)
try
{
if (!isVideo && createThumbnails && thumbnails is not null && !thumbnail)
{
thumbnail = thumbnails.CreateFromFile(sha, path);
}
}
else if (createThumbnails && thumbnails is not null && !thumbnail && frames is not null)
{
var frame = await frames.ExtractJpegAsync(path, ct).ConfigureAwait(false);
if (frame is not null)
else if (isVideo && createThumbnails && thumbnails is not null && !thumbnail && frames is not null)
{
using var stream = new MemoryStream(frame);
thumbnail = thumbnails.Create(sha, stream);
var frame = await frames.ExtractJpegAsync(path, ct).ConfigureAwait(false);
if (frame is not null)
{
using var stream = new MemoryStream(frame);
thumbnail = thumbnails.Create(sha, stream);
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
thumbnail = false; // the neutral tile until a thumbnail can be made
}
return new FileCheck(true, bytes, sha, phash, width, height, thumbnail);
}
private static string Reason(Exception ex) => ex is UnauthorizedAccessException ? "permesso negato" : ex.Message;
private static Entry BuildEntry(long accountId, Item item)
{
var e = item.Last.Entry;
@@ -429,6 +529,23 @@ public sealed class LibraryRebuilder(
return media with { State = MediaState.Deleted };
}
var imported = string.Equals(item.Last.Entry.Provenance, "imported", StringComparison.OrdinalIgnoreCase);
if (check is { Unreadable: { } reason })
{
// The file is where the journal says: it stays linked as recorded there (nothing to download again), and
// the report says it could not be verified.
report.FilesUnreadable++;
report.Note($"File illeggibile, ripristinato senza verifica: {file.Path} ({reason})");
return media with
{
State = imported ? MediaState.Imported : MediaState.Downloaded,
RelativePath = LibraryLayout.ToStored(file.Path),
HasThumbnail = check.HasThumbnail,
DownloadedAt = item.Last.At,
LastError = $"File illeggibile durante la ricostruzione del database: {reason}",
};
}
if (check is not { Exists: true })
{
report.FilesMissing++;
@@ -443,7 +560,6 @@ public sealed class LibraryRebuilder(
report.Note($"Contenuto diverso da quello registrato: {file.Path}");
}
var imported = string.Equals(item.Last.Entry.Provenance, "imported", StringComparison.OrdinalIgnoreCase);
return media with
{
State = imported ? MediaState.Imported : MediaState.Downloaded,
@@ -523,10 +639,24 @@ public sealed class LibraryRebuilder(
batch.Clear();
}
/// <summary>A file of a series removed by Alberto after it was saved: known, settled, never downloaded again.</summary>
private static Media BuildRemoved(JournalFile file) => new()
{
Position = file.N,
Kind = string.Equals(file.Type, "video", StringComparison.OrdinalIgnoreCase) ? MediaKind.Video : MediaKind.Photo,
InstagramId = file.Id,
Width = file.Width,
Height = file.Height,
Duration = file.DurationMs is { } ms ? TimeSpan.FromMilliseconds(ms) : null,
Sha256 = file.Sha256 is { Length: 64 } s ? s.ToLowerInvariant() : null,
Bytes = file.Bytes,
State = MediaState.Deleted,
};
private static void CountUntracked(string folder, List<Item> items, Accumulator report)
{
var known = new HashSet<string>(
items.SelectMany(i => i.Files.Values).Select(f => LibraryLayout.ToStored(f.Path)),
items.SelectMany(i => i.Files.Values.Concat(i.Removed.Values)).Select(f => LibraryLayout.ToStored(f.Path)),
OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal);
foreach (var file in Directory.EnumerateFiles(folder, "*", new EnumerationOptions { RecurseSubdirectories = true, IgnoreInaccessible = true }))
{
@@ -548,6 +678,13 @@ public sealed class LibraryRebuilder(
private sealed record FileCheck(bool Exists, long Bytes, string Sha256, ulong? PerceptualHash, int? Width, int? Height, bool HasThumbnail)
{
public static FileCheck Missing { get; } = new(false, 0, string.Empty, null, null, null, false);
/// <summary>Set when the file is on disk but could not be read: why.</summary>
public string? Unreadable { get; init; }
public bool IsMissing => !Exists && Unreadable is null;
public static FileCheck CannotRead(string reason) => Missing with { Unreadable = reason };
}
private sealed class Item(EntryKind kind, JournalLine first)
@@ -560,8 +697,14 @@ public sealed class LibraryRebuilder(
public string? InstagramId => string.IsNullOrWhiteSpace(Last.Entry.Id) ? null : Last.Entry.Id.Trim();
/// <summary>An Instagram id or a code: the entry is more than its files.</summary>
public bool HasIdentity => InstagramId is not null || !string.IsNullOrWhiteSpace(Last.Entry.Code);
public SortedDictionary<int, JournalFile> Files { get; } = [];
/// <summary>Positions listed by an earlier save line and no longer by the last one: removed by Alberto.</summary>
public SortedDictionary<int, JournalFile> Removed { get; } = [];
public bool Deleted { get; set; }
public DateTimeOffset? DeletedAt { get; set; }
@@ -589,6 +732,8 @@ public sealed class LibraryRebuilder(
public int FilesMissing { get; set; }
public int FilesUnreadable { get; set; }
public int HashMismatches { get; set; }
public int UntrackedFiles { get; set; }
@@ -614,6 +759,7 @@ public sealed class LibraryRebuilder(
EntriesDeleted = EntriesDeleted,
FilesVerified = FilesVerified,
FilesMissing = FilesMissing,
FilesUnreadable = FilesUnreadable,
HashMismatches = HashMismatches,
UntrackedFiles = UntrackedFiles,
InvalidLines = InvalidLines,
@@ -0,0 +1,175 @@
using System.Security.Cryptography;
using Febe.Core.Domain;
using Febe.Core.Import;
using Febe.Core.Library;
using Febe.Tests.Import.Support;
using Febe.Tests.Support;
namespace Febe.Tests.Import;
/// <summary>«Tieni il più grande» when the source file is larger: it replaces the library file (rule 12, spec §9.3).</summary>
[TestClass]
public sealed class ImportReplacementTests
{
/// <summary>
/// Two versions of the same photo in the source (the original and a smaller export), both paired with one smaller
/// library file and both decided «Tieni il più grande»: only the largest takes its place, whatever the order of
/// the files, and with «Sposta» the larger original is never lost.
/// </summary>
[TestMethod]
[DataRow(ImportMode.Copy, true)]
[DataRow(ImportMode.Move, true)]
[DataRow(ImportMode.Copy, false)]
[DataRow(ImportMode.Move, false)]
public async Task TwoSourceFilesPairedWithTheSameLibraryFileLeaveOnlyTheLargest(ImportMode mode, bool largestFirst)
{
using var temp = new TempFolder("importa");
await using var bed = await ImportTestBed.CreateAsync(temp);
var (marta, library) = await bed.AccountWithPhotoAsync("marta.demo", "60", ImportTestFiles.Resized(60, 800, 600, 400, 300, 70));
var largest = ImportTestFiles.Photo(60, 800, 600, 92);
var export = ImportTestFiles.Resized(60, 800, 600, 600, 450, 80);
var largestPath = bed.Write(largestFirst ? "marta.demo/1-originale.jpg" : "marta.demo/2-originale.jpg", largest);
var exportPath = bed.Write(largestFirst ? "marta.demo/2-esportata.jpg" : "marta.demo/1-esportata.jpg", export);
var run = await bed.AnalyzeAsync();
await bed.Review.SetModeAsync(run.Id, mode);
var cases = await bed.Review.ListDuplicatesAsync(run.Id);
Assert.HasCount(2, cases);
Assert.IsTrue(cases.All(c => c.Existing.MediaId == library.Media[0].Id), "Both source files are paired with the library file.");
Assert.AreEqual(2, await bed.Review.DecideAsync(run.Id, cases[0].Pair.Id, DuplicateDecision.KeepLarger, DecisionScope.AllPending));
var plan = await bed.Review.GetPlanAsync(run.Id);
Assert.AreEqual(1, plan.ToReplace, "One library file, one replacement.");
Assert.AreEqual(1, plan.ToImport);
await bed.Copier.CopyAsync(run.Id);
var media = (await bed.Library.Db.Media.GetAsync(library.Media[0].Id))!;
var largestSha = Sha(largest);
Assert.AreEqual(largestSha, media.Sha256, "The library keeps the largest version.");
Assert.AreEqual((800, 600), (media.Width, media.Height));
var onDisk = bed.Library.Layout.FullPath(marta, media.RelativePath!);
Assert.AreEqual(largestSha, Sha(File.ReadAllBytes(onDisk)), "The record points to a file that exists and holds the largest version.");
var folder = bed.Library.Layout.AccountFolder(marta);
Assert.HasCount(1, Directory.GetFiles(folder, "*.jpg", SearchOption.AllDirectories), "One version in the library: no leftovers.");
var files = await bed.FilesAsync(run.Id);
var winner = files.Single(f => f.Sha256 == largestSha);
Assert.AreEqual(mode == ImportMode.Move ? ImportFileState.Moved : ImportFileState.Copied, winner.State, winner.LastError);
StringAssert.StartsWith(winner.LastError, ImportText.NoteReplaced);
Assert.AreEqual(media.RelativePath, winner.DestinationPath);
var loser = files.Single(f => f.Sha256 == Sha(export));
Assert.AreEqual(ImportFileState.Skipped, loser.State, "The smaller export never takes the place of the original.");
Assert.AreEqual(ImportText.NoteLargerInSource, loser.LastError);
Assert.AreEqual(mode == ImportMode.Copy, File.Exists(largestPath), "With «Sposta» the original goes only once its copy is verified.");
Assert.IsTrue(File.Exists(exportPath), "A skipped file is never deleted, not even with «Sposta».");
CollectionAssert.AreEqual(mode == ImportMode.Move ? new[] { largestPath } : Array.Empty<string>(), bed.FileSystem.Deleted);
var journal = await AccountJournal.ReadAsync(folder);
Assert.AreEqual(media.RelativePath, journal.Lines.Last(l => l.Entry.Id == "60").Files!.Single().Path);
// Running the copy again changes nothing.
await bed.Copier.CopyAsync(run.Id);
Assert.AreEqual(largestSha, (await bed.Library.Db.Media.GetAsync(media.Id))!.Sha256);
Assert.HasCount(1, Directory.GetFiles(folder, "*.jpg", SearchOption.AllDirectories));
}
/// <summary>
/// The plan compares sizes before the copy starts; the copy looks again at the file in the library right before
/// replacing it, and a larger file that arrived meanwhile stays.
/// </summary>
[TestMethod]
public async Task AReplacementNeverRemovesALargerFileThatArrivedMeanwhile()
{
using var temp = new TempFolder("importa");
await using var bed = await ImportTestBed.CreateAsync(temp);
var (marta, library) = await bed.AccountWithPhotoAsync("marta.demo", "62", ImportTestFiles.Resized(62, 800, 600, 400, 300, 70));
bed.Write("marta.demo/a-altra.jpg", ImportTestFiles.Photo(63));
var candidate = bed.Write("marta.demo/b-grande.jpg", ImportTestFiles.Photo(62, 800, 600, 92));
var run = await bed.AnalyzeAsync();
var pair = (await bed.Review.ListDuplicatesAsync(run.Id)).Single();
await bed.Review.DecideAsync(run.Id, pair.Pair.Id, DuplicateDecision.KeepLarger);
Assert.AreEqual(1, (await bed.Review.GetPlanAsync(run.Id)).ToReplace);
// Right after the first file, a larger version of the photo takes the place of the library file.
var huge = ImportTestFiles.Photo(62, 1600, 1200, 92);
var hugeRelative = "post/arrivata-nel-frattempo.jpg";
var hugePath = bed.Library.Layout.FullPath(marta, hugeRelative);
var swapped = false;
var progress = new SyncProgress(p =>
{
if (swapped || p.Phase != ImportPhase.Copying || p.CurrentFile != "marta.demo/a-altra.jpg")
{
return;
}
swapped = true;
Directory.CreateDirectory(Path.GetDirectoryName(hugePath)!);
File.WriteAllBytes(hugePath, huge);
bed.Library.Db.Media.MarkStoredAsync(
library.Media[0].Id,
new StoredFile(hugeRelative, huge.Length, Sha(huge), null, 1600, 1200, null, false),
MediaState.Downloaded,
bed.Library.Clock.Now).GetAwaiter().GetResult();
});
await bed.Copier.CopyAsync(run.Id, progress);
Assert.IsTrue(swapped);
var media = (await bed.Library.Db.Media.GetAsync(library.Media[0].Id))!;
Assert.AreEqual(Sha(huge), media.Sha256, "The larger file stays in the library.");
Assert.IsTrue(File.Exists(hugePath));
var file = await bed.FileAsync(run.Id, "marta.demo/b-grande.jpg");
Assert.AreEqual(ImportFileState.Skipped, file.State);
Assert.AreEqual(ImportText.NoteLibraryLarger, file.LastError);
Assert.IsTrue(File.Exists(candidate));
}
/// <summary>
/// An imported entry without Instagram id (with only a code, or with nothing but its file) whose file is replaced
/// by a larger one: after losing the database the rebuild finds one entry with the new file, on every run.
/// </summary>
[TestMethod]
[DataRow(false)]
[DataRow(true)]
public async Task AReplacedImportedFileIsOneEntryAfterARebuild(bool withCode)
{
using var temp = new TempFolder("importa");
await using var bed = await ImportTestBed.CreateAsync(temp);
var marta = await bed.Library.Db.Accounts.AddAsync(Make.Account("marta.demo", "900"));
var name = withCode ? "2026-10-10_18-22-05_DBx3kQ2_1.jpg" : "IMG_0001.jpg";
var small = bed.Write("marta.demo/" + name, ImportTestFiles.Resized(64, 800, 600, 400, 300, 70));
await bed.Copier.CopyAsync((await bed.AnalyzeAsync()).Id);
var first = (await bed.Library.Db.Entries.ListByAccountAsync(marta.Id)).Single();
Assert.AreEqual(withCode ? "DBx3kQ2" : null, first.Code);
Assert.IsNull(first.InstagramId);
File.Delete(small);
bed.Write("marta.demo/originali/" + name, ImportTestFiles.Photo(64, 800, 600, 92));
var run = await bed.AnalyzeAsync();
var pair = (await bed.Review.ListDuplicatesAsync(run.Id)).Single();
await bed.Review.DecideAsync(run.Id, pair.Pair.Id, DuplicateDecision.KeepLarger);
await bed.Copier.CopyAsync(run.Id);
var replaced = (await bed.Library.Db.Media.ListByEntryAsync(first.Id)).Single();
Assert.AreEqual((800, 600), (replaced.Width, replaced.Height), "Replaced by the larger file.");
await bed.Library.LoseDatabaseAsync();
var rebuilder = new LibraryRebuilder(bed.Library.Db, bed.Library.Layout, bed.Library.Clock);
var report = await rebuilder.RebuildAsync();
Assert.AreEqual(1, report.EntriesCreated, string.Join("; ", report.Notes));
Assert.AreEqual(0, report.FilesMissing, "The replaced file is not reported as missing.");
var account = (await bed.Library.Db.Accounts.FindByInstagramIdAsync("900"))!;
var entry = (await bed.Library.Db.Entries.ListByAccountAsync(account.Id)).Single();
Assert.AreEqual(replaced.RelativePath, (await bed.Library.Db.Media.ListByEntryAsync(entry.Id)).Single().RelativePath);
Assert.AreEqual(0, (await rebuilder.RebuildAsync()).EntriesCreated, "A second run adds nothing.");
Assert.HasCount(1, await bed.Library.Db.Entries.ListByAccountAsync(account.Id));
}
private static string Sha(byte[] bytes) => Convert.ToHexStringLower(SHA256.HashData(bytes));
private sealed class SyncProgress(Action<ImportProgress> report) : IProgress<ImportProgress>
{
public void Report(ImportProgress value) => report(value);
}
}
@@ -89,6 +89,35 @@ public sealed class MediaReaderTests
Assert.IsNull(Mp4Reader.Read(new MemoryStream("not a movie at all"u8.ToArray()), Now));
}
/// <summary>
/// A 64-bit box size close to 2^63 overflows «offset + size»: the reader must stop on it, not walk backwards
/// forever (the analysis would hang and every «Riprendi» would hit the same file again).
/// </summary>
[TestMethod]
public void AHugeSixtyFourBitBoxSizeEndsTheReadingInsteadOfLooping()
{
static byte[] Box(string type, params byte[][] parts) =>
[.. BigEndian((uint)(8 + parts.Sum(p => p.Length))), .. Encoding.ASCII.GetBytes(type), .. parts.SelectMany(p => p)];
static byte[] BigEndian(uint value) => [(byte)(value >> 24), (byte)(value >> 16), (byte)(value >> 8), (byte)value];
byte[] huge = [0, 0, 0, 1, .. "xxxx"u8, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xF8];
var ftyp = Box("ftyp", "isom"u8.ToArray(), [0, 0, 2, 0]);
// Inside moov: a valid «free» box, then the damaged one.
byte[] insideMoov = [.. ftyp, .. Box("moov", Box("free"), huge)];
// At the top level, after ftyp.
byte[] topLevel = [.. ftyp, .. huge, .. new byte[32]];
foreach (var (name, bytes) in new[] { ("moov", insideMoov), ("primo livello", topLevel) })
{
var reading = Task.Run(() => Mp4Reader.Read(new MemoryStream(bytes), Now));
Assert.IsTrue(reading.Wait(TimeSpan.FromSeconds(10)), $"{name}: la lettura deve finire.");
var info = reading.Result;
Assert.IsNull(info?.CreatedAt, name);
Assert.IsNull(info?.Width, name);
}
}
[TestMethod]
public void AnMp4MadeByFfmpegCarriesItsCreationTime()
{
+79 -2
View File
@@ -52,9 +52,14 @@ public sealed class DailyBackupTests
}
await File.WriteAllTextAsync(Path.Combine(folder, "appunti.txt"), "non toccare");
await File.WriteAllTextAsync(Path.Combine(folder, "febe-2026-10-13.sqlite.abcd1234.part"), "copia interrotta");
var clock = new ManualClock(LibraryTimes.Noon(today));
var interrupted = Path.Combine(folder, "febe-2026-10-12.sqlite.abcd1234.part");
await File.WriteAllTextAsync(interrupted, "copia interrotta");
File.SetLastWriteTimeUtc(interrupted, clock.Now.UtcDateTime.AddDays(-1));
var writing = Path.Combine(folder, "febe-2026-10-13.sqlite.ef567890.part");
await File.WriteAllTextAsync(writing, "copia in corso");
File.SetLastWriteTimeUtc(writing, clock.Now.UtcDateTime.AddMinutes(-2));
var backup = new DailyBackup(db.Maintenance, clock, folder);
var result = await backup.RunAsync();
@@ -64,9 +69,81 @@ public sealed class DailyBackupTests
Assert.AreEqual(today, kept[0]);
Assert.AreEqual(today.AddDays(-6), kept[^1]);
Assert.HasCount(5, result.Removed); // four old copies and the interrupted one
Assert.IsFalse(File.Exists(interrupted));
Assert.IsTrue(File.Exists(writing), "A temporary copy written minutes ago may belong to a copy still running.");
Assert.IsTrue(File.Exists(Path.Combine(folder, "appunti.txt")));
}
/// <summary>
/// «Fai una copia adesso» clicked twice, or clicked while the 3:20 copy runs: the copies take turns, and neither
/// one removes the temporary file of the other.
/// </summary>
[TestMethod]
public async Task TwoCopiesAtTheSameTimeBothSucceed()
{
using var temp = new TempFolder();
var folder = temp.Combine("dati", "copie");
var clock = new ManualClock(LibraryTimes.Noon(new DateOnly(2026, 10, 13)));
var database = new SlowBackups();
var first = new DailyBackup(database, clock, folder).RunNowAsync();
await database.FirstStarted.Task.WaitAsync(TimeSpan.FromSeconds(10));
var second = new DailyBackup(database, clock, folder).RunNowAsync();
await Task.WhenAny(second, Task.Delay(500)); // without turns, the second copy finishes here and its retention runs
database.ReleaseFirst.SetResult();
var results = await Task.WhenAll(first, second).WaitAsync(TimeSpan.FromSeconds(10));
Assert.IsTrue(results.All(r => r.Created is not null));
Assert.AreEqual(1, database.MaxConcurrent, "One copy of a folder at a time.");
Assert.HasCount(1, Directory.GetFiles(folder, "*.sqlite"));
Assert.IsEmpty(Directory.GetFiles(folder, "*.part"));
}
/// <summary>VACUUM INTO stand-in: the first copy waits until the test lets it finish.</summary>
private sealed class SlowBackups : Core.Storage.IDatabaseMaintenance
{
private int _calls;
private int _running;
private int _max;
public TaskCompletionSource FirstStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public TaskCompletionSource ReleaseFirst { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public int MaxConcurrent => Volatile.Read(ref _max);
public string DatabasePath => "febe.sqlite";
public int SchemaVersion => 1;
public async Task BackupToAsync(string destinationFile, CancellationToken ct = default)
{
var running = Interlocked.Increment(ref _running);
int seen;
while ((seen = Volatile.Read(ref _max)) < running && Interlocked.CompareExchange(ref _max, running, seen) != seen)
{
}
try
{
await File.WriteAllTextAsync(destinationFile, "copia", ct);
if (Interlocked.Increment(ref _calls) == 1)
{
FirstStarted.SetResult();
await ReleaseFirst.Task.WaitAsync(ct);
}
}
finally
{
Interlocked.Decrement(ref _running);
}
}
public Task<bool> CheckIntegrityAsync(CancellationToken ct = default) => Task.FromResult(true);
public Task OptimizeAsync(CancellationToken ct = default) => Task.CompletedTask;
}
[TestMethod]
public void RecognizesOnlyItsOwnFileNames()
{
@@ -0,0 +1,152 @@
using Febe.Core.Abstractions;
using Febe.Core.Collection;
using Febe.Core.Collection.Downloads;
using Febe.Core.Domain;
using Febe.Core.Import;
using Febe.Core.Library;
using Febe.Core.Storage;
using Febe.Tests.Collection.Support;
using Febe.Tests.Import.Support;
using Febe.Tests.Support;
namespace Febe.Tests.Library;
/// <summary>
/// Every saved entry gets its <c>_febe.jsonl</c> line (spec §8.5), also when the journal cannot be written at that moment
/// (a sync client holding it, a network share that dropped) and when the last file of an entry to settle is not stored.
/// A file already in the library is never treated as «nothing written».
/// </summary>
[TestClass]
public sealed class JournalBacklogTests
{
[TestMethod]
public async Task ADownloadedFileStaysInTheLibraryWhenTheJournalIsBusyAndItsLineFollows()
{
using var temp = new TempFolder("giornale");
await using var lib = await TestLibrary.CreateAsync(temp);
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
await lib.SaveAsync(account, Make.Entry(account.Id, "prima", Make.T0.AddDays(-1)), [TestFile.Photo(1, 3)]);
var entryId = await AddAsync(lib.Db, account, "p1", Make.Photo() with { SourceUrl = "https://cdn.esempio/1.jpg" });
var queue = Queue(lib, temp, _ => (FetchOutcome.Completed, TestFile.PhotoBytes(21, 64, 80)));
var journal = AccountJournal.PathFor(lib.Layout.AccountFolder(account));
DownloadReport report;
await using (new FileStream(journal, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
report = await queue.RunOnceAsync(CancellationToken.None);
}
Assert.AreEqual(1, report.Stored, report.ToString());
var media = (await lib.Db.Media.ListByEntryAsync(entryId)).Single();
Assert.AreEqual(MediaState.Downloaded, media.State, "The file is in the library: it is not to download again.");
Assert.IsTrue(File.Exists(lib.Layout.FullPath(account, media.RelativePath!)));
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.GetAsync(entryId))!.State, "It stays visible in the library.");
Assert.AreEqual(entryId.ToString(System.Globalization.CultureInfo.InvariantCulture), await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
Assert.IsTrue((await lib.Db.Events.ListAfterAsync(0)).Any(e => e.Code == "giornale-in-sospeso"));
Assert.IsFalse((await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Any(l => l.Entry.Id == "p1"));
// The next pass writes the line, even with nothing to download.
await queue.RunOnceAsync(CancellationToken.None);
var line = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Single(l => l.Entry.Id == "p1");
Assert.AreEqual(media.RelativePath, line.Files!.Single().Path);
Assert.IsNull(await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
// So a rebuild after losing the database finds it, without downloading anything.
await lib.LoseDatabaseAsync();
await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock).RebuildAsync();
var again = await lib.Db.Accounts.FindByInstagramIdAsync("1001");
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.FindAsync(again!.Id, EntryKind.Post, "p1"))!.State);
}
[TestMethod]
public async Task AnEntryGetsItsLineAlsoWhenItsLastFileIsSkippedOrFails()
{
using var temp = new TempFolder("giornale");
await using var lib = await TestLibrary.CreateAsync(temp);
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
var tooLarge = await AddAsync(lib.Db, account, "p2", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p2.jpg" }, Make.Video(2) with { SourceUrl = "https://cdn.esempio/grande.mp4" });
var refused = await AddAsync(lib.Db, account, "p3", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p3.jpg" }, Make.Photo(2) with { SourceUrl = "https://cdn.esempio/altrove.jpg" });
var queue = Queue(lib, temp, uri => uri.AbsolutePath switch
{
"/grande.mp4" => (FetchOutcome.Completed, new byte[5000]),
"/altrove.jpg" => (FetchOutcome.HostNotAllowed, null),
"/p2.jpg" => (FetchOutcome.Completed, TestFile.PhotoBytes(31, 64, 80)),
_ => (FetchOutcome.Completed, TestFile.PhotoBytes(32, 64, 80)),
});
var report = await queue.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(2, report.Stored, report.ToString());
Assert.AreEqual(1, report.Skipped);
Assert.AreEqual(1, report.Failed);
var lines = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines;
foreach (var (id, entryId) in new[] { ("p2", tooLarge), ("p3", refused) })
{
var stored = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(m => m.Position == 1);
var line = lines.SingleOrDefault(l => l.Entry.Id == id);
Assert.IsNotNull(line, $"{id}: the entry is settled with a file in the library, so it has its line.");
Assert.AreEqual(stored.RelativePath, line.Files!.Single().Path);
}
}
[TestMethod]
public async Task AnImportedFileStaysCopiedWhenTheJournalIsBusyAndItsLineFollows()
{
using var temp = new TempFolder("giornale");
await using var bed = await ImportTestBed.CreateAsync(temp);
var (marta, _) = await bed.AccountWithPhotoAsync("marta.demo", "300", ImportTestFiles.Photo(30));
bed.Write("marta.demo/nuova.jpg", ImportTestFiles.Photo(31));
var run = await bed.AnalyzeAsync();
var folder = bed.Library.Layout.AccountFolder(marta);
await using (new FileStream(AccountJournal.PathFor(folder), FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
await bed.Copier.CopyAsync(run.Id);
}
var file = await bed.FileAsync(run.Id, "marta.demo/nuova.jpg");
Assert.AreEqual(ImportFileState.Copied, file.State, file.LastError);
Assert.IsNotNull(file.EntryId);
Assert.AreEqual(MediaState.Imported, (await bed.Library.Db.Media.ListByEntryAsync(file.EntryId.Value)).Single().State);
Assert.IsFalse((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
// Any later copy (or download pass) writes the line that is still missing.
await bed.Copier.CopyAsync(run.Id);
Assert.IsTrue((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
Assert.IsNull(await bed.Library.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
Assert.AreEqual(ImportFileState.Copied, (await bed.FileAsync(run.Id, "marta.demo/nuova.jpg")).State);
}
private static DownloadQueue Queue(TestLibrary lib, TempFolder temp, Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) =>
new(lib.Db, lib.Ingest, new TableFetcher(answer), new FakeDataBudget(), lib.Clock, new SeededRandom(1),
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, params Media[] media)
{
var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1));
var added = await db.Entries.AddAsync(entry, media.Select(m => m with { SourceSeenAt = Make.T0 }).ToList());
return added.Stored.Entry.Id;
}
/// <summary>A file fetcher answering from a table: no network at all.</summary>
private sealed class TableFetcher(Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) : IFileFetcher
{
public async Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
{
var (outcome, bytes) = answer(address);
if (bytes is null)
{
return new FetchResult(outcome, 0, null, "errore di prova");
}
if (options?.MaxBytes is { } max && bytes.Length > max)
{
await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct);
return new FetchResult(FetchOutcome.TooLarge, max, 200, "troppo grande");
}
await destination.WriteAsync(bytes, ct);
return new FetchResult(outcome, bytes.Length, 200, null);
}
}
}
@@ -0,0 +1,169 @@
using Febe.Core.Domain;
using Febe.Core.Library;
using Febe.Tests.Support;
namespace Febe.Tests.Library;
/// <summary>
/// «Ricostruisci il database dalla libreria» (spec §8.5) on libraries that changed after their files were saved: files
/// removed or replaced by Alberto's choices, files that cannot be read. The rebuild respects the choices, never stops
/// on one file, and a second run adds nothing.
/// </summary>
[TestClass]
public sealed class LibraryRebuildRecoveryTests
{
/// <summary>One file of a series removed by «Tieni quello in libreria» stays removed, not «file mancante».</summary>
[TestMethod]
public async Task AFileOfASeriesRemovedByADuplicateDecisionStaysRemoved()
{
using var temp = new TempFolder("ricostruzione");
await using var lib = await TestLibrary.CreateAsync(temp);
var account = await lib.Db.Accounts.AddAsync(Make.Account("doppie.prova", "500"));
await lib.SaveAsync(account, Make.Entry(account.Id, "100", Make.T0.AddDays(-3)), [TestFile.Photo(1, 41, 1080, 1350)]);
var series = await lib.SaveAsync(account, Make.Entry(account.Id, "200", Make.T0), [TestFile.Photo(1, 41, 720, 900), TestFile.Photo(2, 77, 1080, 1080)]);
var review = new DownloadDuplicateReview(lib.Db, lib.Layout, lib.Journal, lib.Clock, lib.Thumbnails, lib.Index);
var pair = (await review.ListAsync()).Single(c => c.Candidate.EntryId == series.Entry.Id);
Assert.AreEqual(1, (await review.DecideAsync(pair.Pair.Id, DuplicateDecision.KeepExisting)).FilesRemoved);
var before = (await lib.Db.Entries.GetWithMediaAsync(series.Entry.Id))!;
await lib.LoseDatabaseAsync();
var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails);
var report = await rebuilder.RebuildAsync();
Assert.AreEqual(0, report.FilesMissing, string.Join("; ", report.Notes));
var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("500"))!;
var entry = (await lib.Db.Entries.FindAsync(accountAgain.Id, EntryKind.Post, "200"))!;
var media = await lib.Db.Media.ListByEntryAsync(entry.Id);
Assert.AreEqual(MediaState.Deleted, media.Single(m => m.Position == 1).State, "Alberto's removal is respected: never looked for or downloaded again.");
Assert.AreEqual(MediaState.Downloaded, media.Single(m => m.Position == 2).State);
Assert.AreEqual(before.Media.Single(m => m.Position == 2).RelativePath, media.Single(m => m.Position == 2).RelativePath);
Assert.AreEqual(before.Entry.State, entry.State);
var again = await rebuilder.RebuildAsync();
Assert.AreEqual(0, again.EntriesCreated);
}
/// <summary>
/// A journal written before replacements closed the old file: an imported entry without identifier appears twice,
/// once with the file that was replaced. Only the entry with its file comes back, on every run.
/// </summary>
[TestMethod]
public async Task AnEntryWithoutIdentifierWhoseFileWasReplacedComesBackOnce()
{
using var temp = new TempFolder("ricostruzione");
await using var lib = await TestLibrary.CreateAsync(temp);
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "600"));
var folder = lib.Layout.AccountFolder(account);
var bytes = TestFile.PhotoBytes(5, 800, 600);
Directory.CreateDirectory(Path.Combine(folder, "importati"));
await File.WriteAllBytesAsync(Path.Combine(folder, "importati", "IMG_1 (2).jpg"), bytes);
var entry = new Entry
{
AccountId = account.Id,
Kind = EntryKind.Post,
PublishedAt = Make.T0.AddDays(-400),
Provenance = Provenance.Imported,
DiscoveredAt = Make.T0,
};
Media Imported(string path, string sha) => Make.Photo() with { State = MediaState.Imported, RelativePath = path, Sha256 = sha, Bytes = 100 };
await lib.Journal.AppendSavedAsync(folder, account, entry, [Imported("importati/IMG_1.jpg", Make.Sha(1))]);
await lib.Journal.AppendSavedAsync(folder, account, entry, [Imported("importati/IMG_1 (2).jpg", Make.Sha(2))]);
await lib.LoseDatabaseAsync();
var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock);
var report = await rebuilder.RebuildAsync();
Assert.AreEqual(1, report.EntriesCreated);
Assert.AreEqual(1, report.FilesMissing, "The old file is reported...");
Assert.IsTrue(report.Notes.Any(n => n.Contains("importati/IMG_1.jpg", StringComparison.Ordinal) && n.Contains("non ripristinato", StringComparison.Ordinal)));
var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("600"))!;
var entries = await lib.Db.Entries.ListByAccountAsync(accountAgain.Id);
Assert.HasCount(1, entries, "...but not restored as an entry of its own.");
Assert.AreEqual("importati/IMG_1 (2).jpg", (await lib.Db.Media.ListByEntryAsync(entries[0].Id)).Single().RelativePath);
var again = await rebuilder.RebuildAsync();
Assert.AreEqual(0, again.EntriesCreated, "A second run adds nothing.");
Assert.HasCount(1, await lib.Db.Entries.ListByAccountAsync(accountAgain.Id));
}
/// <summary>A remembered deletion of an imported entry without identifier is not added again by every run.</summary>
[TestMethod]
public async Task ARememberedDeletionWithoutIdentifierIsRestoredOnce()
{
using var temp = new TempFolder("ricostruzione");
await using var lib = await TestLibrary.CreateAsync(temp);
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "700"));
var imported = await lib.SaveAsync(account, Make.Entry(account.Id, "x", Make.T0.AddDays(-300)) with { InstagramId = null, Code = null, Provenance = Provenance.Imported }, [TestFile.Photo(1, 9)]);
await new LibraryDeletion(lib.Db, lib.Layout, lib.Journal, lib.Clock).DeleteAsync(imported.Entry.Id, DeletionMode.Remember);
await lib.LoseDatabaseAsync();
var rebuilder = new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock);
Assert.AreEqual(1, (await rebuilder.RebuildAsync()).EntriesDeleted);
var again = await rebuilder.RebuildAsync();
Assert.AreEqual(0, again.EntriesCreated);
var accountAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("700"))!;
var entries = await lib.Db.Entries.ListByAccountAsync(accountAgain.Id);
Assert.HasCount(1, entries);
Assert.AreEqual(EntryState.Deleted, entries[0].State);
Assert.AreEqual(imported.Media[0].Sha256, (await lib.Db.Media.ListByEntryAsync(entries[0].Id)).Single().Sha256, "The SHA-256 is remembered: an import does not bring it back.");
}
/// <summary>
/// A file that cannot be read (here: held exclusively by another program) and an unreadable journal do not stop
/// the rebuild of everything else; the file stays linked as the journal recorded it.
/// </summary>
[TestMethod]
public async Task AnUnreadableFileOrJournalDoesNotStopTheRebuild()
{
using var temp = new TempFolder("ricostruzione");
await using var lib = await TestLibrary.CreateAsync(temp);
var marta = await lib.Db.Accounts.AddAsync(Make.Account("marta.demo", "800"));
var luca = await lib.Db.Accounts.AddAsync(Make.Account("luca.demo", "801"));
var zoe = await lib.Db.Accounts.AddAsync(Make.Account("zoe.demo", "802"));
var locked = await lib.SaveAsync(marta, Make.Entry(marta.Id, "a1", Make.T0.AddDays(-2)), [TestFile.Photo(1, 11)]);
var fine = await lib.SaveAsync(marta, Make.Entry(marta.Id, "a2", Make.T0.AddDays(-1)), [TestFile.Photo(1, 12)]);
await lib.SaveAsync(luca, Make.Entry(luca.Id, "b1", Make.T0), [TestFile.Photo(1, 13)]);
await lib.SaveAsync(zoe, Make.Entry(zoe.Id, "c1", Make.T0), [TestFile.Photo(1, 14)]);
await lib.LoseDatabaseAsync();
var lockedPath = lib.Layout.FullPath(marta, locked.Media[0].RelativePath!);
var report = await WhileHeldAsync(
[lockedPath, AccountJournal.PathFor(lib.Layout.AccountFolder(luca))],
() => new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails).RebuildAsync());
Assert.AreEqual(1, report.FilesUnreadable, string.Join("; ", report.Notes));
Assert.AreEqual(2, report.FilesVerified, "The other files of the folder and of the following folders are verified.");
Assert.IsTrue(report.Notes.Any(n => n.Contains("luca.demo", StringComparison.Ordinal) && n.Contains("_febe.jsonl", StringComparison.Ordinal)));
var martaAgain = (await lib.Db.Accounts.FindByInstagramIdAsync("800"))!;
var restored = (await lib.Db.Media.ListByEntryAsync((await lib.Db.Entries.FindAsync(martaAgain.Id, EntryKind.Post, "a1"))!.Id)).Single();
Assert.AreEqual(MediaState.Downloaded, restored.State, "The file is where the journal says: linked again, not to be downloaded.");
Assert.AreEqual(locked.Media[0].RelativePath, restored.RelativePath);
Assert.AreEqual(locked.Media[0].Sha256, restored.Sha256);
StringAssert.Contains(restored.LastError, "illeggibile");
Assert.IsNotNull(await lib.Db.Entries.FindAsync(martaAgain.Id, EntryKind.Post, "a2"));
Assert.IsNotNull(await lib.Db.Accounts.FindByInstagramIdAsync("802"), "The folders after the unreadable one are rebuilt too.");
Assert.IsNull(await lib.Db.Accounts.FindByInstagramIdAsync("801"), "The folder with the unreadable journal waits for the next run.");
var later = await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock, lib.Thumbnails).RebuildAsync();
Assert.AreEqual(1, later.EntriesCreated, "Once readable, the skipped folder is rebuilt.");
_ = fine;
}
/// <summary>Runs <paramref name="action"/> while other handles hold the files with no sharing at all.</summary>
private static async Task<T> WhileHeldAsync<T>(IReadOnlyList<string> paths, Func<Task<T>> action)
{
var handles = paths.Select(p => new FileStream(p, FileMode.Open, FileAccess.ReadWrite, FileShare.None)).ToList();
try
{
return await action();
}
finally
{
foreach (var handle in handles)
{
await handle.DisposeAsync();
}
}
}
}