Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
153 lines
8.4 KiB
C#
153 lines
8.4 KiB
C#
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
using Febe.Core.Collection.Downloads;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Import;
|
|
using Febe.Core.Library;
|
|
using Febe.Core.Storage;
|
|
using Febe.Tests.Collection.Support;
|
|
using Febe.Tests.Import.Support;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Library;
|
|
|
|
/// <summary>
|
|
/// Every saved entry gets its <c>_febe.jsonl</c> line (spec §8.5), also when the journal cannot be written at that moment
|
|
/// (a sync client holding it, a network share that dropped) and when the last file of an entry to settle is not stored.
|
|
/// A file already in the library is never treated as «nothing written».
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class JournalBacklogTests
|
|
{
|
|
[TestMethod]
|
|
public async Task ADownloadedFileStaysInTheLibraryWhenTheJournalIsBusyAndItsLineFollows()
|
|
{
|
|
using var temp = new TempFolder("giornale");
|
|
await using var lib = await TestLibrary.CreateAsync(temp);
|
|
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
await lib.SaveAsync(account, Make.Entry(account.Id, "prima", Make.T0.AddDays(-1)), [TestFile.Photo(1, 3)]);
|
|
var entryId = await AddAsync(lib.Db, account, "p1", Make.Photo() with { SourceUrl = "https://cdn.esempio/1.jpg" });
|
|
var queue = Queue(lib, temp, _ => (FetchOutcome.Completed, TestFile.PhotoBytes(21, 64, 80)));
|
|
var journal = AccountJournal.PathFor(lib.Layout.AccountFolder(account));
|
|
|
|
DownloadReport report;
|
|
await using (new FileStream(journal, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
|
|
{
|
|
report = await queue.RunOnceAsync(CancellationToken.None);
|
|
}
|
|
|
|
Assert.AreEqual(1, report.Stored, report.ToString());
|
|
var media = (await lib.Db.Media.ListByEntryAsync(entryId)).Single();
|
|
Assert.AreEqual(MediaState.Downloaded, media.State, "The file is in the library: it is not to download again.");
|
|
Assert.IsTrue(File.Exists(lib.Layout.FullPath(account, media.RelativePath!)));
|
|
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.GetAsync(entryId))!.State, "It stays visible in the library.");
|
|
Assert.AreEqual(entryId.ToString(System.Globalization.CultureInfo.InvariantCulture), await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
|
|
Assert.IsTrue((await lib.Db.Events.ListAfterAsync(0)).Any(e => e.Code == "giornale-in-sospeso"));
|
|
Assert.IsFalse((await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Any(l => l.Entry.Id == "p1"));
|
|
|
|
// The next pass writes the line, even with nothing to download.
|
|
await queue.RunOnceAsync(CancellationToken.None);
|
|
var line = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Single(l => l.Entry.Id == "p1");
|
|
Assert.AreEqual(media.RelativePath, line.Files!.Single().Path);
|
|
Assert.IsNull(await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
|
|
|
|
// So a rebuild after losing the database finds it, without downloading anything.
|
|
await lib.LoseDatabaseAsync();
|
|
await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock).RebuildAsync();
|
|
var again = await lib.Db.Accounts.FindByInstagramIdAsync("1001");
|
|
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.FindAsync(again!.Id, EntryKind.Post, "p1"))!.State);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnEntryGetsItsLineAlsoWhenItsLastFileIsSkippedOrFails()
|
|
{
|
|
using var temp = new TempFolder("giornale");
|
|
await using var lib = await TestLibrary.CreateAsync(temp);
|
|
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
|
|
var tooLarge = await AddAsync(lib.Db, account, "p2", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p2.jpg" }, Make.Video(2) with { SourceUrl = "https://cdn.esempio/grande.mp4" });
|
|
var refused = await AddAsync(lib.Db, account, "p3", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p3.jpg" }, Make.Photo(2) with { SourceUrl = "https://cdn.esempio/altrove.jpg" });
|
|
var queue = Queue(lib, temp, uri => uri.AbsolutePath switch
|
|
{
|
|
"/grande.mp4" => (FetchOutcome.Completed, new byte[5000]),
|
|
"/altrove.jpg" => (FetchOutcome.HostNotAllowed, null),
|
|
"/p2.jpg" => (FetchOutcome.Completed, TestFile.PhotoBytes(31, 64, 80)),
|
|
_ => (FetchOutcome.Completed, TestFile.PhotoBytes(32, 64, 80)),
|
|
});
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
|
|
Assert.AreEqual(2, report.Stored, report.ToString());
|
|
Assert.AreEqual(1, report.Skipped);
|
|
Assert.AreEqual(1, report.Failed);
|
|
var lines = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines;
|
|
foreach (var (id, entryId) in new[] { ("p2", tooLarge), ("p3", refused) })
|
|
{
|
|
var stored = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(m => m.Position == 1);
|
|
var line = lines.SingleOrDefault(l => l.Entry.Id == id);
|
|
Assert.IsNotNull(line, $"{id}: the entry is settled with a file in the library, so it has its line.");
|
|
Assert.AreEqual(stored.RelativePath, line.Files!.Single().Path);
|
|
}
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnImportedFileStaysCopiedWhenTheJournalIsBusyAndItsLineFollows()
|
|
{
|
|
using var temp = new TempFolder("giornale");
|
|
await using var bed = await ImportTestBed.CreateAsync(temp);
|
|
var (marta, _) = await bed.AccountWithPhotoAsync("marta.demo", "300", ImportTestFiles.Photo(30));
|
|
bed.Write("marta.demo/nuova.jpg", ImportTestFiles.Photo(31));
|
|
var run = await bed.AnalyzeAsync();
|
|
var folder = bed.Library.Layout.AccountFolder(marta);
|
|
|
|
await using (new FileStream(AccountJournal.PathFor(folder), FileMode.Open, FileAccess.ReadWrite, FileShare.None))
|
|
{
|
|
await bed.Copier.CopyAsync(run.Id);
|
|
}
|
|
|
|
var file = await bed.FileAsync(run.Id, "marta.demo/nuova.jpg");
|
|
Assert.AreEqual(ImportFileState.Copied, file.State, file.LastError);
|
|
Assert.IsNotNull(file.EntryId);
|
|
Assert.AreEqual(MediaState.Imported, (await bed.Library.Db.Media.ListByEntryAsync(file.EntryId.Value)).Single().State);
|
|
Assert.IsFalse((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
|
|
|
|
// Any later copy (or download pass) writes the line that is still missing.
|
|
await bed.Copier.CopyAsync(run.Id);
|
|
Assert.IsTrue((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
|
|
Assert.IsNull(await bed.Library.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
|
|
Assert.AreEqual(ImportFileState.Copied, (await bed.FileAsync(run.Id, "marta.demo/nuova.jpg")).State);
|
|
}
|
|
|
|
private static DownloadQueue Queue(TestLibrary lib, TempFolder temp, Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) =>
|
|
new(lib.Db, lib.Ingest, new TableFetcher(answer), new FakeDataBudget(), lib.Clock, new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
|
|
|
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, params Media[] media)
|
|
{
|
|
var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1));
|
|
var added = await db.Entries.AddAsync(entry, media.Select(m => m with { SourceSeenAt = Make.T0 }).ToList());
|
|
return added.Stored.Entry.Id;
|
|
}
|
|
|
|
/// <summary>A file fetcher answering from a table: no network at all.</summary>
|
|
private sealed class TableFetcher(Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) : IFileFetcher
|
|
{
|
|
public async Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
|
|
{
|
|
var (outcome, bytes) = answer(address);
|
|
if (bytes is null)
|
|
{
|
|
return new FetchResult(outcome, 0, null, "errore di prova");
|
|
}
|
|
|
|
if (options?.MaxBytes is { } max && bytes.Length > max)
|
|
{
|
|
await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct);
|
|
return new FetchResult(FetchOutcome.TooLarge, max, 200, "troppo grande");
|
|
}
|
|
|
|
await destination.WriteAsync(bytes, ct);
|
|
return new FetchResult(outcome, bytes.Length, 200, null);
|
|
}
|
|
}
|
|
}
|