Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Mzbb6zeFTjT6Rp8b92AVkA
474 lines
26 KiB
C#
474 lines
26 KiB
C#
using System.Globalization;
|
|
using System.Text.Json.Nodes;
|
|
using Febe.Browser.Download;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
using Febe.Core.Collection.Downloads;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Engine;
|
|
using Febe.Core.Library;
|
|
using Febe.Core.Storage;
|
|
using Febe.Tests.Browser.TestSupport;
|
|
using Febe.Tests.Collection.Support;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// The download queue (spec §6.5) without a browser: stop signals of the CDN, failures that may pass, addresses refused
|
|
/// again after their renewal, entries deleted while their files download, identical files downloaded together, the
|
|
/// clean-up of an interrupted run, and the data budget of files over the size limit.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class DownloadQueueTests
|
|
{
|
|
[TestMethod]
|
|
public async Task A429OfTheCdnIsAThrottleSignalThatStopsTheQueueWithoutCountingAnAttempt()
|
|
{
|
|
using var temp = new TempFolder("coda-429");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var files = new List<long>();
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
files.Add(await AddAsync(library.Db, account, $"p{i}", EntryKind.Post, $"https://cdn.esempio/m/{i}.jpg?firma=segreta"));
|
|
}
|
|
|
|
var breaker = new RecordingBreaker();
|
|
var fetcher = ScriptedFetcher.Status(429);
|
|
var queue = Queue(library, temp, fetcher, new SignalDetector(SignalRules.LoadDefault(), breaker, library.Clock), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, fetcher.Calls, "The pass ends at the first 429: the other files are not asked for.");
|
|
var signal = breaker.Signals.Single();
|
|
Assert.AreEqual(SignalLevel.Throttle, signal.Level);
|
|
Assert.AreEqual("http-429", signal.Code);
|
|
Assert.AreEqual("https://cdn.esempio/", signal.Url, "Only the host: the path of a CDN address identifies a file.");
|
|
Assert.IsNotNull(report.PausedUntil);
|
|
Assert.IsTrue(report.PausedUntil >= library.Clock.Now.AddMinutes(59), report.PausedUntil.ToString());
|
|
foreach (var id in files)
|
|
{
|
|
var media = (await library.Db.Media.ListByEntryAsync(id)).Single();
|
|
Assert.AreEqual(MediaState.ToDownload, media.State);
|
|
Assert.AreEqual(0, media.Attempts, "The CDN asked to slow down: not the file's fault.");
|
|
}
|
|
|
|
// Until the pause is over the queue asks the CDN for nothing, however often it is woken.
|
|
queue.Wake();
|
|
Assert.IsNotNull((await queue.RunOnceAsync(CancellationToken.None)).PausedUntil);
|
|
Assert.AreEqual(1, fetcher.Calls);
|
|
|
|
library.Clock.Now = report.PausedUntil!.Value.AddMinutes(1);
|
|
fetcher.Answer = ScriptedFetcher.PhotoByPath;
|
|
var after = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(3, after.Stored, after.ToString());
|
|
Assert.AreEqual(1, breaker.Signals.Count);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void ADownloadStatusIsReportedOncePerWindowButAlwaysRecognized()
|
|
{
|
|
var clock = new VirtualClock();
|
|
var breaker = new RecordingBreaker();
|
|
var signals = new SignalDetector(SignalRules.LoadDefault(), breaker, clock);
|
|
var address = new Uri("https://cdn.esempio/v/t51/abc.jpg?oh=firma");
|
|
Assert.AreEqual(SignalLevel.Throttle, signals.ObserveDownload(address, 429));
|
|
Assert.AreEqual(SignalLevel.Throttle, signals.ObserveDownload(address, 429), "Still a throttle, to stop the caller.");
|
|
Assert.AreEqual(1, breaker.Signals.Count, "Reported once in the dedupe window.");
|
|
Assert.IsNull(signals.ObserveDownload(address, 404));
|
|
Assert.IsNull(signals.ObserveDownload(address, 500));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFailureThatMayPassIsRetriedLaterAndLaterBeforeTheFileFails()
|
|
{
|
|
using var temp = new TempFolder("coda-attesa");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var fetcher = ScriptedFetcher.Network("net::ERR_CONNECTION_RESET");
|
|
var queue = Queue(library, temp, fetcher);
|
|
var start = library.Clock.Now;
|
|
|
|
Assert.AreEqual(1, (await queue.RunOnceAsync(CancellationToken.None)).Retried);
|
|
Assert.AreEqual(0, (await queue.RunOnceAsync(CancellationToken.None)).Total, "Not again at the next pass: it waits.");
|
|
Assert.AreEqual(1, fetcher.Calls);
|
|
|
|
// One pass a minute, as when idle, until the file is settled.
|
|
var passes = 0;
|
|
while ((await library.Db.Media.ListByEntryAsync(entry)).Single().State == MediaState.ToDownload && passes++ < 2000)
|
|
{
|
|
library.Clock.Now = library.Clock.Now.AddMinutes(1);
|
|
await queue.RunOnceAsync(CancellationToken.None);
|
|
}
|
|
|
|
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
Assert.AreEqual(MediaState.Failed, media.State);
|
|
Assert.AreEqual(5, fetcher.Calls, "Five attempts in all.");
|
|
Assert.IsTrue(library.Clock.Now - start >= TimeSpan.FromHours(10.5), $"Spread over {library.Clock.Now - start}, not over a few passes.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task WithoutAConnectionNoAttemptIsCountedAndThePassEnds()
|
|
{
|
|
using var temp = new TempFolder("coda-rete");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entries = new List<long>();
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
entries.Add(await AddAsync(library.Db, account, $"p{i}", EntryKind.Story, $"https://cdn.esempio/m/{i}.jpg"));
|
|
}
|
|
|
|
var fetcher = ScriptedFetcher.Network("net::ERR_INTERNET_DISCONNECTED");
|
|
var queue = Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, fetcher.Calls, "Every file would fail the same way: the pass ends.");
|
|
Assert.IsNotNull(report.PausedUntil);
|
|
foreach (var id in entries)
|
|
{
|
|
Assert.AreEqual(0, (await library.Db.Media.ListByEntryAsync(id)).Single().Attempts, "The router restarting is not the file's fault.");
|
|
}
|
|
|
|
library.Clock.Now = report.PausedUntil!.Value.AddSeconds(1);
|
|
fetcher.Answer = ScriptedFetcher.PhotoByPath;
|
|
Assert.AreEqual(3, (await queue.RunOnceAsync(CancellationToken.None)).Stored, "Back online: the stories are still there to download.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(30_000, CooperativeCancellation = true)]
|
|
public async Task ManyWakeCallsDuringAPassGiveOneMorePassNotOneEach()
|
|
{
|
|
using var temp = new TempFolder("coda-sveglia");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
|
|
// Every pass tries the file again (no wait between attempts here): the passes are the attempts.
|
|
var fetcher = ScriptedFetcher.Status(503);
|
|
var queue = new DownloadQueue(library.Db, library.Ingest, fetcher, new FakeDataBudget(), new IdleClock(), new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), RetryDelays = [], MaxAttempts = 1000 });
|
|
for (var i = 0; i < 50; i++)
|
|
{
|
|
queue.Wake(); // e.g. one per «ricetta.*» event of a long session
|
|
}
|
|
|
|
using var stop = new CancellationTokenSource();
|
|
var running = queue.RunAsync(stop.Token);
|
|
await Task.Delay(TimeSpan.FromMilliseconds(500));
|
|
await stop.CancelAsync();
|
|
await Assert.ThrowsAsync<OperationCanceledException>(() => running);
|
|
Assert.AreEqual(2, fetcher.Calls, "The pass, then one more for all the calls: then it waits.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task FilesWaitingForTheirNextAttemptDoNotHideTheOthers()
|
|
{
|
|
using var temp = new TempFolder("coda-finestra");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
await AddAsync(library.Db, account, "a", EntryKind.Post, "https://cdn.esempio/m/a.jpg");
|
|
await AddAsync(library.Db, account, "b", EntryKind.Post, "https://cdn.esempio/m/b.jpg");
|
|
var good = await AddAsync(library.Db, account, "c", EntryKind.Post, "https://cdn.esempio/m/c.jpg");
|
|
var fetcher = new ScriptedFetcher((uri, destination, ct) => uri!.AbsolutePath.EndsWith("/c.jpg", StringComparison.Ordinal)
|
|
? ScriptedFetcher.Photo(31)(uri, destination, ct)
|
|
: Task.FromResult(new FetchResult(FetchOutcome.HttpError, 0, 503, "prova")));
|
|
var queue = Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), BatchSize = 2 });
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(2, report.Retried, report.ToString());
|
|
Assert.AreEqual(1, report.Stored, "The third file is reached in the same pass.");
|
|
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(good)).Single().State);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnAddressRefusedAgainAfterItsRenewalsEndsFailedWithoutAnotherSinglePost()
|
|
{
|
|
using var temp = new TempFolder("coda-rinnovo");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var renewed = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg", attempts: 4);
|
|
var fresh = await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
|
|
var queue = Queue(library, temp, ScriptedFetcher.Status(403));
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Failed, report.ToString());
|
|
Assert.AreEqual(1, report.Expired);
|
|
var refused = (await library.Db.Media.ListByEntryAsync(renewed)).Single();
|
|
Assert.AreEqual(MediaState.Failed, refused.State);
|
|
StringAssert.Contains(refused.LastError, "rinnovo");
|
|
Assert.AreEqual(MediaState.Expired, (await library.Db.Media.ListByEntryAsync(fresh)).Single().State);
|
|
var tasks = (await library.Db.Tasks.ListPendingAsync(10)).Where(t => t.Kind == EngineTaskKind.SinglePost).ToList();
|
|
CollectionAssert.AreEqual(new[] { fresh.ToString(CultureInfo.InvariantCulture) }, tasks.Select(t => t.Payload).ToArray(), "«Post singolo» only for the first expiry.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnEntryDeletedWhileItsFileDownloadsStaysDeletedAndOutOfTheLibrary()
|
|
{
|
|
using var temp = new TempFolder("coda-eliminato");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var deletion = new LibraryDeletion(library.Db, library.Layout, library.Journal, library.Clock, library.Thumbnails, library.Index);
|
|
|
|
// Alberto deletes the entry («ricorda la scelta») while its file is on its way.
|
|
var fetcher = new ScriptedFetcher(async (uri, destination, ct) =>
|
|
{
|
|
await deletion.DeleteAsync(entry, DeletionMode.Remember, ct);
|
|
return await ScriptedFetcher.Photo(41)(uri, destination, ct);
|
|
});
|
|
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
|
|
|
|
Assert.AreEqual(0, report.Stored, report.ToString());
|
|
Assert.AreEqual(EntryState.Deleted, (await library.Db.Entries.GetAsync(entry))!.State);
|
|
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
Assert.AreEqual(MediaState.Deleted, media.State);
|
|
Assert.IsNull(media.RelativePath);
|
|
var folder = library.Layout.AccountFolder(account);
|
|
Assert.AreEqual(0, Directory.EnumerateFiles(folder, "*.jpg", SearchOption.AllDirectories).Count(), "Nothing entered the library.");
|
|
var lines = (await AccountJournal.ReadAsync(folder)).Lines;
|
|
Assert.AreEqual(JournalOps.Delete, lines[^1].Op, "No «save» line after the «delete» line: a rebuild keeps it deleted.");
|
|
Assert.AreEqual(0, lines.Count(l => l.Op == JournalOps.Save));
|
|
Assert.AreEqual(0, Directory.EnumerateFiles(temp.Combine("dati", "temporanei")).Count());
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnEntryRemovedWhileItsFileDownloadsLeavesNoOrphanAndTheQueueGoesOn()
|
|
{
|
|
using var temp = new TempFolder("coda-tolto");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var other = await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
|
|
var deletion = new LibraryDeletion(library.Db, library.Layout, library.Journal, library.Clock, library.Thumbnails, library.Index);
|
|
var fetcher = new ScriptedFetcher(async (uri, destination, ct) =>
|
|
{
|
|
if (uri!.AbsolutePath.EndsWith("/1.jpg", StringComparison.Ordinal))
|
|
{
|
|
await deletion.DeleteAsync(entry, DeletionMode.AllowReturn, ct); // «Eliminalo e lascia che torni»
|
|
}
|
|
|
|
return await ScriptedFetcher.PhotoByPath(uri, destination, ct);
|
|
});
|
|
|
|
var report = await Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 }).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Stored, "The pass goes on with the next file.");
|
|
Assert.IsNull(await library.Db.Entries.GetAsync(entry));
|
|
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(other)).Single().State);
|
|
Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "No orphan file for the removed entry.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFileOfADeletedEntryIsNeverRecordedAsStored()
|
|
{
|
|
using var temp = new TempFolder("coda-registro");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
await library.Db.Entries.MarkDeletedAsync(entry, library.Clock.Now);
|
|
|
|
var recorded = await library.Db.Media.MarkStoredAsync(media.Id, new StoredFile("post/2026/x_1.jpg", 10, Make.Sha(1)), MediaState.Downloaded, library.Clock.Now);
|
|
Assert.IsFalse(recorded);
|
|
var after = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
Assert.AreEqual(MediaState.Deleted, after.State);
|
|
Assert.IsNull(after.RelativePath);
|
|
Assert.AreEqual(EntryState.Deleted, (await library.Db.Entries.GetAsync(entry))!.State);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TwoIdenticalFilesDownloadedAtTheSameTimeEnterTheLibraryOnce()
|
|
{
|
|
using var temp = new TempFolder("coda-gemelli");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
|
|
var bytes = TestFile.PhotoBytes(17, 900, 1100);
|
|
using var together = new Barrier(2);
|
|
var fetcher = new ScriptedFetcher(async (_, destination, ct) =>
|
|
{
|
|
await Task.Run(() => together.SignalAndWait(TimeSpan.FromSeconds(10), ct), ct);
|
|
await destination.WriteAsync(bytes, ct);
|
|
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
|
|
});
|
|
|
|
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Stored, report.ToString());
|
|
Assert.AreEqual(1, report.Duplicates);
|
|
Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "§9.5: one copy of the same bytes per account.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheCleanupOfAnInterruptedRunLeavesTheFilesOnTheirWayAlone()
|
|
{
|
|
using var temp = new TempFolder("coda-pulizia");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var staging = temp.Combine("dati", "temporanei");
|
|
Directory.CreateDirectory(staging);
|
|
var leftover = Path.Combine(staging, SafeFileWriter.TempPrefix + "vecchio" + SafeFileWriter.TempSuffix);
|
|
await File.WriteAllTextAsync(leftover, "lasciato da un arresto");
|
|
var removed = -1;
|
|
|
|
// The start-up clean-up runs while the first download is being written.
|
|
var bytes = TestFile.PhotoBytes(23, 640, 800);
|
|
var fetcher = new ScriptedFetcher(async (_, destination, ct) =>
|
|
{
|
|
await destination.WriteAsync(bytes.AsMemory(0, 1000), ct);
|
|
removed = SafeFileWriter.CleanupTemporaryFiles(staging, recursive: false);
|
|
await destination.WriteAsync(bytes.AsMemory(1000), ct);
|
|
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
|
|
});
|
|
|
|
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, removed, "Only the file of the previous run.");
|
|
Assert.IsFalse(File.Exists(leftover));
|
|
Assert.AreEqual(1, report.Stored, report.ToString());
|
|
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(entry)).Single().State);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheCleanupLeavesALibraryFileBeingWrittenAlone()
|
|
{
|
|
using var temp = new TempFolder("pulizia-libreria");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var added = await library.Db.Entries.AddAsync(Make.Entry(account.Id, "p1", Make.T0.AddDays(-1)), [Make.Photo()]);
|
|
var folder = library.Layout.AccountFolder(account);
|
|
var removed = -1;
|
|
var content = new InterceptingStream(TestFile.PhotoBytes(29, 640, 800), () => removed = SafeFileWriter.CleanupTemporaryFiles(folder));
|
|
|
|
var result = await library.Ingest.SaveMediaAsync(account, added.Stored.Entry, added.Stored.Media[0], new MediaContent { Content = content, Extension = ".jpg" });
|
|
Assert.AreEqual(0, removed, "The temporary file of the write in progress is not a leftover.");
|
|
Assert.AreEqual(MediaSaveOutcome.Stored, result.Outcome);
|
|
Assert.IsTrue(File.Exists(library.Layout.FullPath(account, result.File!.RelativePath)));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFileOverTheLimitCountsWhatTheBrowserLoadedNotWhatWasWritten()
|
|
{
|
|
using var temp = new TempFolder("coda-limite");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
|
|
await library.Db.Entries.AddAsync(Make.Entry(account.Id, "v1", Make.T0), [Make.Video() with { SourceUrl = "https://cdn.esempio/m/v.mp4", SourceSeenAt = Make.T0 }]);
|
|
var budget = new FakeDataBudget();
|
|
var fetcher = new ScriptedFetcher((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.TooLarge, 0, 200, "troppo grande") { TransferredBytes = 600_000_000 }));
|
|
|
|
var report = await Queue(library, temp, fetcher, budget: budget).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Skipped);
|
|
Assert.AreEqual(600_000_000, budget.Used, "The daily data budget counts the real transfer.");
|
|
Assert.AreEqual(600_000_000, report.Bytes);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheBrowserFetcherDoesNotReadAFileThatDeclaresMoreThanTheLimit()
|
|
{
|
|
var cdp = new FakeCdpSession(new VirtualClock());
|
|
cdp.Responder = (method, _) => method switch
|
|
{
|
|
"Page.getFrameTree" => new JsonObject { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1" } } },
|
|
"Network.loadNetworkResource" => new JsonObject
|
|
{
|
|
["resource"] = new JsonObject
|
|
{
|
|
["success"] = true,
|
|
["httpStatusCode"] = 200,
|
|
["stream"] = "S1",
|
|
["headers"] = new JsonObject { ["Content-Type"] = "video/mp4", ["content-length"] = "600000" },
|
|
},
|
|
},
|
|
"IO.read" => new JsonObject { ["data"] = Convert.ToBase64String(new byte[10_000]), ["base64Encoded"] = true, ["eof"] = false },
|
|
_ => null,
|
|
};
|
|
var fetcher = new BrowserFileFetcher(cdp, new HostClassifier(["127.0.0.1:5000"], ["127.0.0.1:5001"]));
|
|
using var destination = new MemoryStream();
|
|
|
|
var result = await fetcher.FetchAsync(new Uri("http://127.0.0.1:5001/grande.mp4"), destination, new FetchOptions(MaxBytes: 100_000), CancellationToken.None);
|
|
Assert.AreEqual(FetchOutcome.TooLarge, result.Outcome);
|
|
Assert.AreEqual(0, destination.Length);
|
|
Assert.AreEqual(600_000, result.TransferredBytes);
|
|
Assert.AreEqual(600_000, result.BudgetBytes);
|
|
Assert.AreEqual(0, cdp.Sent.Count(c => c.Method == "IO.read"), "Nothing more to read: the browser already loaded it.");
|
|
Assert.AreEqual(1, cdp.Sent.Count(c => c.Method == "IO.close"));
|
|
}
|
|
|
|
private static DownloadQueue Queue(TestLibrary library, TempFolder temp, IFileFetcher fetcher, SignalDetector? signals = null, DownloadOptions? options = null, IDataBudget? budget = null) =>
|
|
new(library.Db, library.Ingest, fetcher, budget ?? new FakeDataBudget(), library.Clock, new SeededRandom(1), options ?? new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei") }, signals);
|
|
|
|
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, EntryKind kind, string url, int attempts = 0)
|
|
{
|
|
var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1)) with
|
|
{
|
|
Kind = kind,
|
|
ExpiresAt = kind == EntryKind.Story ? Make.T0.AddHours(20) : null,
|
|
};
|
|
var added = await db.Entries.AddAsync(entry, [Make.Photo() with { SourceUrl = url, SourceSeenAt = Make.T0, Attempts = attempts }]);
|
|
return added.Stored.Entry.Id;
|
|
}
|
|
|
|
/// <summary>A file fetcher answering from a function that can be changed between passes: no network.</summary>
|
|
private sealed class ScriptedFetcher(Func<Uri?, Stream, CancellationToken, Task<FetchResult>> answer) : IFileFetcher
|
|
{
|
|
private int _calls;
|
|
|
|
public Func<Uri?, Stream, CancellationToken, Task<FetchResult>> Answer { get; set; } = answer;
|
|
|
|
public int Calls => Volatile.Read(ref _calls);
|
|
|
|
public static ScriptedFetcher Status(int status) =>
|
|
new((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.HttpError, 0, status, $"Il server ha risposto {status}.")));
|
|
|
|
public static ScriptedFetcher Network(string error) =>
|
|
new((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.NetworkError, 0, null, error)));
|
|
|
|
/// <summary>A photo of scene <paramref name="scene"/>, written whole.</summary>
|
|
public static Func<Uri?, Stream, CancellationToken, Task<FetchResult>> Photo(int scene) => async (_, destination, ct) =>
|
|
{
|
|
var bytes = TestFile.PhotoBytes(scene, 320, 400);
|
|
await destination.WriteAsync(bytes, ct);
|
|
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
|
|
};
|
|
|
|
/// <summary>A photo whose scene depends on the address: different files, different bytes.</summary>
|
|
public static Task<FetchResult> PhotoByPath(Uri? address, Stream destination, CancellationToken ct) =>
|
|
Photo(100 + (address?.AbsolutePath ?? string.Empty).Aggregate(0, (sum, c) => (sum * 31) + c) % 1000)(address, destination, ct);
|
|
|
|
public Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
|
|
{
|
|
Interlocked.Increment(ref _calls);
|
|
return Answer(address, destination, ct);
|
|
}
|
|
}
|
|
|
|
/// <summary>A clock whose short waits (pauses between files) pass at once and whose long ones (idle) never end.</summary>
|
|
private sealed class IdleClock : IClock
|
|
{
|
|
public DateTimeOffset Now => Make.T0;
|
|
|
|
public TimeZoneInfo LocalZone => TestZones.Rome;
|
|
|
|
public Task Delay(TimeSpan time, CancellationToken ct) =>
|
|
time < TimeSpan.FromSeconds(10) ? Task.CompletedTask : Task.Delay(Timeout.Infinite, ct);
|
|
}
|
|
|
|
/// <summary>A content stream that runs <paramref name="midway"/> once, half way through.</summary>
|
|
private sealed class InterceptingStream(byte[] bytes, Action midway) : MemoryStream(bytes)
|
|
{
|
|
private bool _done;
|
|
|
|
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
|
|
{
|
|
var read = await base.ReadAsync(buffer[..Math.Min(buffer.Length, 4096)], cancellationToken);
|
|
if (!_done && Position > Length / 2)
|
|
{
|
|
_done = true;
|
|
midway();
|
|
}
|
|
|
|
return read;
|
|
}
|
|
}
|
|
}
|