Files

153 lines
8.4 KiB
C#

using Febe.Core.Abstractions;
using Febe.Core.Collection;
using Febe.Core.Collection.Downloads;
using Febe.Core.Domain;
using Febe.Core.Import;
using Febe.Core.Library;
using Febe.Core.Storage;
using Febe.Tests.Collection.Support;
using Febe.Tests.Import.Support;
using Febe.Tests.Support;
namespace Febe.Tests.Library;
/// <summary>
/// Every saved entry gets its <c>_febe.jsonl</c> line (spec §8.5), also when the journal cannot be written at that moment
/// (a sync client holding it, a network share that dropped) and when the last file of an entry to settle is not stored.
/// A file already in the library is never treated as «nothing written».
/// </summary>
[TestClass]
public sealed class JournalBacklogTests
{
[TestMethod]
public async Task ADownloadedFileStaysInTheLibraryWhenTheJournalIsBusyAndItsLineFollows()
{
using var temp = new TempFolder("giornale");
await using var lib = await TestLibrary.CreateAsync(temp);
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
await lib.SaveAsync(account, Make.Entry(account.Id, "prima", Make.T0.AddDays(-1)), [TestFile.Photo(1, 3)]);
var entryId = await AddAsync(lib.Db, account, "p1", Make.Photo() with { SourceUrl = "https://cdn.esempio/1.jpg" });
var queue = Queue(lib, temp, _ => (FetchOutcome.Completed, TestFile.PhotoBytes(21, 64, 80)));
var journal = AccountJournal.PathFor(lib.Layout.AccountFolder(account));
DownloadReport report;
await using (new FileStream(journal, FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
report = await queue.RunOnceAsync(CancellationToken.None);
}
Assert.AreEqual(1, report.Stored, report.ToString());
var media = (await lib.Db.Media.ListByEntryAsync(entryId)).Single();
Assert.AreEqual(MediaState.Downloaded, media.State, "The file is in the library: it is not to download again.");
Assert.IsTrue(File.Exists(lib.Layout.FullPath(account, media.RelativePath!)));
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.GetAsync(entryId))!.State, "It stays visible in the library.");
Assert.AreEqual(entryId.ToString(System.Globalization.CultureInfo.InvariantCulture), await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
Assert.IsTrue((await lib.Db.Events.ListAfterAsync(0)).Any(e => e.Code == "giornale-in-sospeso"));
Assert.IsFalse((await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Any(l => l.Entry.Id == "p1"));
// The next pass writes the line, even with nothing to download.
await queue.RunOnceAsync(CancellationToken.None);
var line = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines.Single(l => l.Entry.Id == "p1");
Assert.AreEqual(media.RelativePath, line.Files!.Single().Path);
Assert.IsNull(await lib.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
// So a rebuild after losing the database finds it, without downloading anything.
await lib.LoseDatabaseAsync();
await new LibraryRebuilder(lib.Db, lib.Layout, lib.Clock).RebuildAsync();
var again = await lib.Db.Accounts.FindByInstagramIdAsync("1001");
Assert.AreEqual(EntryState.Downloaded, (await lib.Db.Entries.FindAsync(again!.Id, EntryKind.Post, "p1"))!.State);
}
[TestMethod]
public async Task AnEntryGetsItsLineAlsoWhenItsLastFileIsSkippedOrFails()
{
using var temp = new TempFolder("giornale");
await using var lib = await TestLibrary.CreateAsync(temp);
var account = await lib.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
var tooLarge = await AddAsync(lib.Db, account, "p2", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p2.jpg" }, Make.Video(2) with { SourceUrl = "https://cdn.esempio/grande.mp4" });
var refused = await AddAsync(lib.Db, account, "p3", Make.Photo(1) with { SourceUrl = "https://cdn.esempio/p3.jpg" }, Make.Photo(2) with { SourceUrl = "https://cdn.esempio/altrove.jpg" });
var queue = Queue(lib, temp, uri => uri.AbsolutePath switch
{
"/grande.mp4" => (FetchOutcome.Completed, new byte[5000]),
"/altrove.jpg" => (FetchOutcome.HostNotAllowed, null),
"/p2.jpg" => (FetchOutcome.Completed, TestFile.PhotoBytes(31, 64, 80)),
_ => (FetchOutcome.Completed, TestFile.PhotoBytes(32, 64, 80)),
});
var report = await queue.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(2, report.Stored, report.ToString());
Assert.AreEqual(1, report.Skipped);
Assert.AreEqual(1, report.Failed);
var lines = (await AccountJournal.ReadAsync(lib.Layout.AccountFolder(account))).Lines;
foreach (var (id, entryId) in new[] { ("p2", tooLarge), ("p3", refused) })
{
var stored = (await lib.Db.Media.ListByEntryAsync(entryId)).Single(m => m.Position == 1);
var line = lines.SingleOrDefault(l => l.Entry.Id == id);
Assert.IsNotNull(line, $"{id}: the entry is settled with a file in the library, so it has its line.");
Assert.AreEqual(stored.RelativePath, line.Files!.Single().Path);
}
}
[TestMethod]
public async Task AnImportedFileStaysCopiedWhenTheJournalIsBusyAndItsLineFollows()
{
using var temp = new TempFolder("giornale");
await using var bed = await ImportTestBed.CreateAsync(temp);
var (marta, _) = await bed.AccountWithPhotoAsync("marta.demo", "300", ImportTestFiles.Photo(30));
bed.Write("marta.demo/nuova.jpg", ImportTestFiles.Photo(31));
var run = await bed.AnalyzeAsync();
var folder = bed.Library.Layout.AccountFolder(marta);
await using (new FileStream(AccountJournal.PathFor(folder), FileMode.Open, FileAccess.ReadWrite, FileShare.None))
{
await bed.Copier.CopyAsync(run.Id);
}
var file = await bed.FileAsync(run.Id, "marta.demo/nuova.jpg");
Assert.AreEqual(ImportFileState.Copied, file.State, file.LastError);
Assert.IsNotNull(file.EntryId);
Assert.AreEqual(MediaState.Imported, (await bed.Library.Db.Media.ListByEntryAsync(file.EntryId.Value)).Single().State);
Assert.IsFalse((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
// Any later copy (or download pass) writes the line that is still missing.
await bed.Copier.CopyAsync(run.Id);
Assert.IsTrue((await AccountJournal.ReadAsync(folder)).Lines.Any(l => l.Files?.Any(f => f.Path == file.DestinationPath) == true));
Assert.IsNull(await bed.Library.Db.Settings.GetAsync(LibraryIngest.PendingJournalKey));
Assert.AreEqual(ImportFileState.Copied, (await bed.FileAsync(run.Id, "marta.demo/nuova.jpg")).State);
}
private static DownloadQueue Queue(TestLibrary lib, TempFolder temp, Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) =>
new(lib.Db, lib.Ingest, new TableFetcher(answer), new FakeDataBudget(), lib.Clock, new SeededRandom(1),
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, params Media[] media)
{
var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1));
var added = await db.Entries.AddAsync(entry, media.Select(m => m with { SourceSeenAt = Make.T0 }).ToList());
return added.Stored.Entry.Id;
}
/// <summary>A file fetcher answering from a table: no network at all.</summary>
private sealed class TableFetcher(Func<Uri, (FetchOutcome Outcome, byte[]? Bytes)> answer) : IFileFetcher
{
public async Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
{
var (outcome, bytes) = answer(address);
if (bytes is null)
{
return new FetchResult(outcome, 0, null, "errore di prova");
}
if (options?.MaxBytes is { } max && bytes.Length > max)
{
await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct);
return new FetchResult(FetchOutcome.TooLarge, max, 200, "troppo grande");
}
await destination.WriteAsync(bytes, ct);
return new FetchResult(outcome, bytes.Length, 200, null);
}
}
}