Files
Febe/tests/Febe.Tests/Collection/DownloadQueueTests.cs
T
Alby96andClaude Opus 5.5 0775a1460f
verifica / controlli (push) Successful in 25s
verifica / prove (push) Successful in 25m9s
verifica / installatore (push) Successful in 7m37s
verifica / immagine (push) Successful in 20m21s
Prove che cadevano con il runner carico (#46)
Le prove cadevano a ogni carico della CI, ogni volta prove diverse. Dietro alcune c'erano
difetti veri: FebeDatabase lasciava i file aperti dopo DisposeAsync (comandi SQLite non
chiusi, pool di lettura chiuso dopo lo scrittore), e il WAL spariva durante lo spostamento
della cartella Dati; la scheda dell'account senza statistiche leggeva i file di tutta la
libreria; l'arresto di FebeApp non aspettava i lavori aggiunti da un avvio in corso.
Chromium ha 120 s per avviarsi e aprire la prima scheda, e il ricollegamento smette appena
il processo cambia.

Nelle prove i limiti di tempo diventano protezioni contro un blocco: attese per condizione
con un aiuto comune, limiti per classe nella prova di scala con il tempo migliore, porta 0 e
stderr di Chromium nel browser delle prove, e la prova di «Scollega» di WebView2 controlla
che il processo esca davvero. Provato su Windows con tutti i core occupati.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 19:50:40 +01:00

506 lines
28 KiB
C#

using System.Globalization;
using System.Text.Json.Nodes;
using Febe.Browser.Download;
using Febe.Core.Abstractions;
using Febe.Core.Collection;
using Febe.Core.Collection.Downloads;
using Febe.Core.Domain;
using Febe.Core.Engine;
using Febe.Core.Library;
using Febe.Core.Storage;
using Febe.Tests.Browser.TestSupport;
using Febe.Tests.Collection.Support;
using Febe.Tests.Support;
namespace Febe.Tests.Collection;
/// <summary>
/// The download queue (spec §6.5) without a browser: stop signals of the CDN, failures that may pass, addresses refused
/// again after their renewal, entries deleted while their files download, identical files downloaded together, the
/// clean-up of an interrupted run, and the data budget of files over the size limit.
/// </summary>
[TestClass]
public sealed class DownloadQueueTests
{
public TestContext TestContext { get; set; } = null!;
[TestMethod]
public async Task A429OfTheCdnIsAThrottleSignalThatStopsTheQueueWithoutCountingAnAttempt()
{
using var temp = new TempFolder("coda-429");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var files = new List<long>();
for (var i = 0; i < 3; i++)
{
files.Add(await AddAsync(library.Db, account, $"p{i}", EntryKind.Post, $"https://cdn.esempio/m/{i}.jpg?firma=segreta"));
}
var breaker = new RecordingBreaker();
var fetcher = ScriptedFetcher.Status(429);
var queue = Queue(library, temp, fetcher, new SignalDetector(SignalRules.LoadDefault(), breaker, library.Clock), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
var report = await queue.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(1, fetcher.Calls, "The pass ends at the first 429: the other files are not asked for.");
var signal = breaker.Signals.Single();
Assert.AreEqual(SignalLevel.Throttle, signal.Level);
Assert.AreEqual("http-429", signal.Code);
Assert.AreEqual("https://cdn.esempio/", signal.Url, "Only the host: the path of a CDN address identifies a file.");
Assert.IsNotNull(report.PausedUntil);
Assert.IsTrue(report.PausedUntil >= library.Clock.Now.AddMinutes(59), report.PausedUntil.ToString());
foreach (var id in files)
{
var media = (await library.Db.Media.ListByEntryAsync(id)).Single();
Assert.AreEqual(MediaState.ToDownload, media.State);
Assert.AreEqual(0, media.Attempts, "The CDN asked to slow down: not the file's fault.");
}
// Until the pause is over the queue asks the CDN for nothing, however often it is woken.
queue.Wake();
Assert.IsNotNull((await queue.RunOnceAsync(CancellationToken.None)).PausedUntil);
Assert.AreEqual(1, fetcher.Calls);
library.Clock.Now = report.PausedUntil!.Value.AddMinutes(1);
fetcher.Answer = ScriptedFetcher.PhotoByPath;
var after = await queue.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(3, after.Stored, after.ToString());
Assert.AreEqual(1, breaker.Signals.Count);
}
[TestMethod]
public void ADownloadStatusIsReportedOncePerWindowButAlwaysRecognized()
{
var clock = new VirtualClock();
var breaker = new RecordingBreaker();
var signals = new SignalDetector(SignalRules.LoadDefault(), breaker, clock);
var address = new Uri("https://cdn.esempio/v/t51/abc.jpg?oh=firma");
Assert.AreEqual(SignalLevel.Throttle, signals.ObserveDownload(address, 429));
Assert.AreEqual(SignalLevel.Throttle, signals.ObserveDownload(address, 429), "Still a throttle, to stop the caller.");
Assert.AreEqual(1, breaker.Signals.Count, "Reported once in the dedupe window.");
Assert.IsNull(signals.ObserveDownload(address, 404));
Assert.IsNull(signals.ObserveDownload(address, 500));
}
[TestMethod]
public async Task AFailureThatMayPassIsRetriedLaterAndLaterBeforeTheFileFails()
{
using var temp = new TempFolder("coda-attesa");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
var fetcher = ScriptedFetcher.Network("net::ERR_CONNECTION_RESET");
var queue = Queue(library, temp, fetcher);
var start = library.Clock.Now;
Assert.AreEqual(1, (await queue.RunOnceAsync(CancellationToken.None)).Retried);
Assert.AreEqual(0, (await queue.RunOnceAsync(CancellationToken.None)).Total, "Not again at the next pass: it waits.");
Assert.AreEqual(1, fetcher.Calls);
// One pass a minute, as when idle, until the file is settled.
var passes = 0;
while ((await library.Db.Media.ListByEntryAsync(entry)).Single().State == MediaState.ToDownload && passes++ < 2000)
{
library.Clock.Now = library.Clock.Now.AddMinutes(1);
await queue.RunOnceAsync(CancellationToken.None);
}
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
Assert.AreEqual(MediaState.Failed, media.State);
Assert.AreEqual(5, fetcher.Calls, "Five attempts in all.");
Assert.IsTrue(library.Clock.Now - start >= TimeSpan.FromHours(10.5), $"Spread over {library.Clock.Now - start}, not over a few passes.");
}
[TestMethod]
public async Task WithoutAConnectionNoAttemptIsCountedAndThePassEnds()
{
using var temp = new TempFolder("coda-rete");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var entries = new List<long>();
for (var i = 0; i < 3; i++)
{
entries.Add(await AddAsync(library.Db, account, $"p{i}", EntryKind.Story, $"https://cdn.esempio/m/{i}.jpg"));
}
var fetcher = ScriptedFetcher.Network("net::ERR_INTERNET_DISCONNECTED");
var queue = Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
var steps = new List<DownloadProgress>();
queue.Progress += (_, step) => steps.Add(step);
var report = await queue.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(1, fetcher.Calls, "Every file would fail the same way: the pass ends.");
Assert.IsNotNull(report.PausedUntil);
var waiting = steps.Single();
Assert.AreEqual(DownloadStep.Retried, waiting.Step, "The diagnostic report says why the file is still waiting (issue #39).");
Assert.AreEqual(DownloadCauses.Connection, waiting.Cause);
foreach (var id in entries)
{
Assert.AreEqual(0, (await library.Db.Media.ListByEntryAsync(id)).Single().Attempts, "The router restarting is not the file's fault.");
}
library.Clock.Now = report.PausedUntil!.Value.AddSeconds(1);
fetcher.Answer = ScriptedFetcher.PhotoByPath;
Assert.AreEqual(3, (await queue.RunOnceAsync(CancellationToken.None)).Stored, "Back online: the stories are still there to download.");
}
[TestMethod]
[Timeout(60_000, CooperativeCancellation = true)]
public async Task ManyWakeCallsDuringAPassGiveOneMorePassNotOneEach()
{
using var temp = new TempFolder("coda-sveglia");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
// Every pass tries the file again (no wait between attempts here): the passes are the attempts.
var fetcher = ScriptedFetcher.Status(503);
var clock = new IdleClock();
var queue = new DownloadQueue(library.Db, library.Ingest, fetcher, new FakeDataBudget(), clock, new SeededRandom(1),
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), RetryDelays = [], MaxAttempts = 1000 });
for (var i = 0; i < 50; i++)
{
queue.Wake(); // e.g. one per «ricetta.*» event of a long session
}
// Every pass ends by starting an idle wait (a wake cuts it short): the second one means two passes are done.
// By condition, not by a time window two passes must fit in (#46).
using var stop = new CancellationTokenSource();
var running = queue.RunAsync(stop.Token);
await clock.SecondIdle.Task.WaitAsync(TimeSpan.FromSeconds(20), TestContext.CancellationToken);
await Task.Delay(TimeSpan.FromMilliseconds(300), TestContext.CancellationToken); // negative check: no third pass
await stop.CancelAsync();
await Assert.ThrowsAsync<OperationCanceledException>(() => running);
Assert.AreEqual(2, fetcher.Calls, "The pass, then one more for all the calls: then it waits.");
Assert.AreEqual(2, clock.IdleWaits, "After the second pass the queue waits: nothing wakes it again.");
}
[TestMethod]
public async Task FilesWaitingForTheirNextAttemptDoNotHideTheOthers()
{
using var temp = new TempFolder("coda-finestra");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
await AddAsync(library.Db, account, "a", EntryKind.Post, "https://cdn.esempio/m/a.jpg");
await AddAsync(library.Db, account, "b", EntryKind.Post, "https://cdn.esempio/m/b.jpg");
var good = await AddAsync(library.Db, account, "c", EntryKind.Post, "https://cdn.esempio/m/c.jpg");
var fetcher = new ScriptedFetcher((uri, destination, ct) => uri!.AbsolutePath.EndsWith("/c.jpg", StringComparison.Ordinal)
? ScriptedFetcher.Photo(31)(uri, destination, ct)
: Task.FromResult(new FetchResult(FetchOutcome.HttpError, 0, 503, "prova")));
var queue = Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), BatchSize = 2 });
var report = await queue.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(2, report.Retried, report.ToString());
Assert.AreEqual(1, report.Stored, "The third file is reached in the same pass.");
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(good)).Single().State);
}
[TestMethod]
public async Task AnAddressRefusedAgainAfterItsRenewalsEndsFailedWithoutAnotherSinglePost()
{
using var temp = new TempFolder("coda-rinnovo");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var renewed = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg", attempts: 4);
var fresh = await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
var queue = Queue(library, temp, ScriptedFetcher.Status(403));
var report = await queue.RunOnceAsync(CancellationToken.None);
Assert.AreEqual(1, report.Failed, report.ToString());
Assert.AreEqual(1, report.Expired);
var refused = (await library.Db.Media.ListByEntryAsync(renewed)).Single();
Assert.AreEqual(MediaState.Failed, refused.State);
StringAssert.Contains(refused.LastError, "rinnovo");
Assert.AreEqual(MediaState.Expired, (await library.Db.Media.ListByEntryAsync(fresh)).Single().State);
var tasks = (await library.Db.Tasks.ListPendingAsync(10)).Where(t => t.Kind == EngineTaskKind.SinglePost).ToList();
CollectionAssert.AreEqual(new[] { fresh.ToString(CultureInfo.InvariantCulture) }, tasks.Select(t => t.Payload).ToArray(), "«Post singolo» only for the first expiry.");
}
[TestMethod]
public async Task AnEntryDeletedWhileItsFileDownloadsStaysDeletedAndOutOfTheLibrary()
{
using var temp = new TempFolder("coda-eliminato");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
var deletion = new LibraryDeletion(library.Db, library.Layout, library.Journal, library.Clock, library.Thumbnails, library.Index);
// Alberto deletes the entry («ricorda la scelta») while its file is on its way.
var fetcher = new ScriptedFetcher(async (uri, destination, ct) =>
{
await deletion.DeleteAsync(entry, DeletionMode.Remember, ct);
return await ScriptedFetcher.Photo(41)(uri, destination, ct);
});
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
Assert.AreEqual(0, report.Stored, report.ToString());
Assert.AreEqual(EntryState.Deleted, (await library.Db.Entries.GetAsync(entry))!.State);
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
Assert.AreEqual(MediaState.Deleted, media.State);
Assert.IsNull(media.RelativePath);
var folder = library.Layout.AccountFolder(account);
Assert.AreEqual(0, Directory.EnumerateFiles(folder, "*.jpg", SearchOption.AllDirectories).Count(), "Nothing entered the library.");
var lines = (await AccountJournal.ReadAsync(folder)).Lines;
Assert.AreEqual(JournalOps.Delete, lines[^1].Op, "No «save» line after the «delete» line: a rebuild keeps it deleted.");
Assert.AreEqual(0, lines.Count(l => l.Op == JournalOps.Save));
Assert.AreEqual(0, Directory.EnumerateFiles(temp.Combine("dati", "temporanei")).Count());
}
[TestMethod]
public async Task AnEntryRemovedWhileItsFileDownloadsLeavesNoOrphanAndTheQueueGoesOn()
{
using var temp = new TempFolder("coda-tolto");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
var other = await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
var deletion = new LibraryDeletion(library.Db, library.Layout, library.Journal, library.Clock, library.Thumbnails, library.Index);
var fetcher = new ScriptedFetcher(async (uri, destination, ct) =>
{
if (uri!.AbsolutePath.EndsWith("/1.jpg", StringComparison.Ordinal))
{
await deletion.DeleteAsync(entry, DeletionMode.AllowReturn, ct); // «Eliminalo e lascia che torni»
}
return await ScriptedFetcher.PhotoByPath(uri, destination, ct);
});
var report = await Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 }).RunOnceAsync(CancellationToken.None);
Assert.AreEqual(1, report.Stored, "The pass goes on with the next file.");
Assert.IsNull(await library.Db.Entries.GetAsync(entry));
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(other)).Single().State);
Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "No orphan file for the removed entry.");
}
[TestMethod]
public async Task AFileOfADeletedEntryIsNeverRecordedAsStored()
{
using var temp = new TempFolder("coda-registro");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
await library.Db.Entries.MarkDeletedAsync(entry, library.Clock.Now);
var recorded = await library.Db.Media.MarkStoredAsync(media.Id, new StoredFile("post/2026/x_1.jpg", 10, Make.Sha(1)), MediaState.Downloaded, library.Clock.Now);
Assert.IsFalse(recorded);
var after = (await library.Db.Media.ListByEntryAsync(entry)).Single();
Assert.AreEqual(MediaState.Deleted, after.State);
Assert.IsNull(after.RelativePath);
Assert.AreEqual(EntryState.Deleted, (await library.Db.Entries.GetAsync(entry))!.State);
}
[TestMethod]
public async Task TwoIdenticalFilesDownloadedAtTheSameTimeEnterTheLibraryOnce()
{
using var temp = new TempFolder("coda-gemelli");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
var bytes = TestFile.PhotoBytes(17, 900, 1100);
using var together = new Barrier(2);
var fetcher = new ScriptedFetcher(async (_, destination, ct) =>
{
await Task.Run(() => together.SignalAndWait(TimeSpan.FromSeconds(10), ct), ct);
await destination.WriteAsync(bytes, ct);
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
});
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
Assert.AreEqual(1, report.Stored, report.ToString());
Assert.AreEqual(1, report.Duplicates);
Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "§9.5: one copy of the same bytes per account.");
}
[TestMethod]
public async Task TheCleanupOfAnInterruptedRunLeavesTheFilesOnTheirWayAlone()
{
using var temp = new TempFolder("coda-pulizia");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
var staging = temp.Combine("dati", "temporanei");
Directory.CreateDirectory(staging);
var leftover = Path.Combine(staging, SafeFileWriter.TempPrefix + "vecchio" + SafeFileWriter.TempSuffix);
await File.WriteAllTextAsync(leftover, "lasciato da un arresto");
var removed = -1;
// The start-up clean-up runs while the first download is being written.
var bytes = TestFile.PhotoBytes(23, 640, 800);
var fetcher = new ScriptedFetcher(async (_, destination, ct) =>
{
await destination.WriteAsync(bytes.AsMemory(0, 1000), ct);
removed = SafeFileWriter.CleanupTemporaryFiles(staging, recursive: false);
await destination.WriteAsync(bytes.AsMemory(1000), ct);
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
});
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
Assert.AreEqual(1, removed, "Only the file of the previous run.");
Assert.IsFalse(File.Exists(leftover));
Assert.AreEqual(1, report.Stored, report.ToString());
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(entry)).Single().State);
}
[TestMethod]
public async Task TheCleanupLeavesALibraryFileBeingWrittenAlone()
{
using var temp = new TempFolder("pulizia-libreria");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
var added = await library.Db.Entries.AddAsync(Make.Entry(account.Id, "p1", Make.T0.AddDays(-1)), [Make.Photo()]);
var folder = library.Layout.AccountFolder(account);
var removed = -1;
var content = new InterceptingStream(TestFile.PhotoBytes(29, 640, 800), () => removed = SafeFileWriter.CleanupTemporaryFiles(folder));
var result = await library.Ingest.SaveMediaAsync(account, added.Stored.Entry, added.Stored.Media[0], new MediaContent { Content = content, Extension = ".jpg" });
Assert.AreEqual(0, removed, "The temporary file of the write in progress is not a leftover.");
Assert.AreEqual(MediaSaveOutcome.Stored, result.Outcome);
Assert.IsTrue(File.Exists(library.Layout.FullPath(account, result.File!.RelativePath)));
}
[TestMethod]
public async Task AFileOverTheLimitCountsWhatTheBrowserLoadedNotWhatWasWritten()
{
using var temp = new TempFolder("coda-limite");
await using var library = await TestLibrary.CreateAsync(temp);
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
await library.Db.Entries.AddAsync(Make.Entry(account.Id, "v1", Make.T0), [Make.Video() with { SourceUrl = "https://cdn.esempio/m/v.mp4", SourceSeenAt = Make.T0 }]);
var budget = new FakeDataBudget();
var fetcher = new ScriptedFetcher((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.TooLarge, 0, 200, "troppo grande") { TransferredBytes = 600_000_000 }));
var report = await Queue(library, temp, fetcher, budget: budget).RunOnceAsync(CancellationToken.None);
Assert.AreEqual(1, report.Skipped);
Assert.AreEqual(600_000_000, budget.Used, "The daily data budget counts the real transfer.");
Assert.AreEqual(600_000_000, report.Bytes);
}
[TestMethod]
public async Task TheBrowserFetcherDoesNotReadAFileThatDeclaresMoreThanTheLimit()
{
var cdp = new FakeCdpSession(new VirtualClock());
cdp.Responder = (method, _) => method switch
{
"Page.getFrameTree" => new JsonObject { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1" } } },
"Network.loadNetworkResource" => new JsonObject
{
["resource"] = new JsonObject
{
["success"] = true,
["httpStatusCode"] = 200,
["stream"] = "S1",
["headers"] = new JsonObject { ["Content-Type"] = "video/mp4", ["content-length"] = "600000" },
},
},
"IO.read" => new JsonObject { ["data"] = Convert.ToBase64String(new byte[10_000]), ["base64Encoded"] = true, ["eof"] = false },
_ => null,
};
var fetcher = new BrowserFileFetcher(cdp, new HostClassifier(["127.0.0.1:5000"], ["127.0.0.1:5001"]));
using var destination = new MemoryStream();
var result = await fetcher.FetchAsync(new Uri("http://127.0.0.1:5001/grande.mp4"), destination, new FetchOptions(MaxBytes: 100_000), CancellationToken.None);
Assert.AreEqual(FetchOutcome.TooLarge, result.Outcome);
Assert.AreEqual(0, destination.Length);
Assert.AreEqual(600_000, result.TransferredBytes);
Assert.AreEqual(600_000, result.BudgetBytes);
Assert.AreEqual(0, cdp.Sent.Count(c => c.Method == "IO.read"), "Nothing more to read: the browser already loaded it.");
Assert.AreEqual(1, cdp.Sent.Count(c => c.Method == "IO.close"));
}
private static DownloadQueue Queue(TestLibrary library, TempFolder temp, IFileFetcher fetcher, SignalDetector? signals = null, DownloadOptions? options = null, IDataBudget? budget = null) =>
new(library.Db, library.Ingest, fetcher, budget ?? new FakeDataBudget(), library.Clock, new SeededRandom(1), options ?? new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei") }, signals);
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, EntryKind kind, string url, int attempts = 0)
{
var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1)) with
{
Kind = kind,
ExpiresAt = kind == EntryKind.Story ? Make.T0.AddHours(20) : null,
};
var added = await db.Entries.AddAsync(entry, [Make.Photo() with { SourceUrl = url, SourceSeenAt = Make.T0, Attempts = attempts }]);
return added.Stored.Entry.Id;
}
/// <summary>A file fetcher answering from a function that can be changed between passes: no network.</summary>
private sealed class ScriptedFetcher(Func<Uri?, Stream, CancellationToken, Task<FetchResult>> answer) : IFileFetcher
{
private int _calls;
public Func<Uri?, Stream, CancellationToken, Task<FetchResult>> Answer { get; set; } = answer;
public int Calls => Volatile.Read(ref _calls);
public static ScriptedFetcher Status(int status) =>
new((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.HttpError, 0, status, $"Il server ha risposto {status}.")));
public static ScriptedFetcher Network(string error) =>
new((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.NetworkError, 0, null, error)));
/// <summary>A photo of scene <paramref name="scene"/>, written whole.</summary>
public static Func<Uri?, Stream, CancellationToken, Task<FetchResult>> Photo(int scene) => async (_, destination, ct) =>
{
var bytes = TestFile.PhotoBytes(scene, 320, 400);
await destination.WriteAsync(bytes, ct);
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
};
/// <summary>A photo whose scene depends on the address: different files, different bytes.</summary>
public static Task<FetchResult> PhotoByPath(Uri? address, Stream destination, CancellationToken ct) =>
Photo(100 + (address?.AbsolutePath ?? string.Empty).Aggregate(0, (sum, c) => (sum * 31) + c) % 1000)(address, destination, ct);
public Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
{
Interlocked.Increment(ref _calls);
return Answer(address, destination, ct);
}
}
/// <summary>A clock whose short waits (pauses between files) pass at once and whose long ones (idle) never end.</summary>
private sealed class IdleClock : IClock
{
private int _idleWaits;
public DateTimeOffset Now => Make.T0;
public TimeZoneInfo LocalZone => TestZones.Rome;
/// <summary>How many idle waits started.</summary>
public int IdleWaits => Volatile.Read(ref _idleWaits);
/// <summary>Completed when the second idle wait starts.</summary>
public TaskCompletionSource SecondIdle { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public Task Delay(TimeSpan time, CancellationToken ct)
{
if (time < TimeSpan.FromSeconds(10))
{
return Task.CompletedTask;
}
if (Interlocked.Increment(ref _idleWaits) == 2)
{
SecondIdle.TrySetResult();
}
return Task.Delay(Timeout.Infinite, ct);
}
}
/// <summary>A content stream that runs <paramref name="midway"/> once, half way through.</summary>
private sealed class InterceptingStream(byte[] bytes, Action midway) : MemoryStream(bytes)
{
private bool _done;
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
{
var read = await base.ReadAsync(buffer[..Math.Min(buffer.Length, 4096)], cancellationToken);
if (!_done && Position > Length / 2)
{
_done = true;
midway();
}
return read;
}
}
}