Dopo una prova dal vivo andata male, Alberto deve poter mandare un solo file che spieghi perché, senza che nessuno veda dati di altre persone. Attività ha ora il pulsante «Diagnostica», che scarica il rapporto delle sessioni degli ultimi 14 giorni: pagine aperte e forma degli indirizzi, risposte per tipo, stato e dimensione con ciò che i lettori ne hanno fatto, struttura stretta delle forme sconosciute, ricette, segnali, permessi negati, file in coda e scaricati, e il motivo di ogni arresto. Nel rapporto entrano solo valori da elenchi chiusi, numeri, orari e forme: mai nomi, indirizzi completi, codici, cursori, cookie o pezzi di risposte. Le forme degli indirizzi e la regola delle chiavi-dato passano dalla sonda a Core, e la sonda le richiama. Il registratore non lancia mai eccezioni verso la raccolta e tiene i file in Dati/diagnostica per 14 giorni. Prova che il rapporto non contiene valori, con controlli di mutazione; tre giri di revisione avversaria (25 difetti confermati, tutti corretti). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
479 lines
27 KiB
C#
479 lines
27 KiB
C#
using System.Globalization;
|
|
using System.Text.Json.Nodes;
|
|
using Febe.Browser.Download;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
using Febe.Core.Collection.Downloads;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Engine;
|
|
using Febe.Core.Library;
|
|
using Febe.Core.Storage;
|
|
using Febe.Tests.Browser.TestSupport;
|
|
using Febe.Tests.Collection.Support;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>
|
|
/// The download queue (spec §6.5) without a browser: stop signals of the CDN, failures that may pass, addresses refused
|
|
/// again after their renewal, entries deleted while their files download, identical files downloaded together, the
|
|
/// clean-up of an interrupted run, and the data budget of files over the size limit.
|
|
/// </summary>
|
|
[TestClass]
|
|
public sealed class DownloadQueueTests
|
|
{
|
|
[TestMethod]
|
|
public async Task A429OfTheCdnIsAThrottleSignalThatStopsTheQueueWithoutCountingAnAttempt()
|
|
{
|
|
using var temp = new TempFolder("coda-429");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var files = new List<long>();
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
files.Add(await AddAsync(library.Db, account, $"p{i}", EntryKind.Post, $"https://cdn.esempio/m/{i}.jpg?firma=segreta"));
|
|
}
|
|
|
|
var breaker = new RecordingBreaker();
|
|
var fetcher = ScriptedFetcher.Status(429);
|
|
var queue = Queue(library, temp, fetcher, new SignalDetector(SignalRules.LoadDefault(), breaker, library.Clock), new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, fetcher.Calls, "The pass ends at the first 429: the other files are not asked for.");
|
|
var signal = breaker.Signals.Single();
|
|
Assert.AreEqual(SignalLevel.Throttle, signal.Level);
|
|
Assert.AreEqual("http-429", signal.Code);
|
|
Assert.AreEqual("https://cdn.esempio/", signal.Url, "Only the host: the path of a CDN address identifies a file.");
|
|
Assert.IsNotNull(report.PausedUntil);
|
|
Assert.IsTrue(report.PausedUntil >= library.Clock.Now.AddMinutes(59), report.PausedUntil.ToString());
|
|
foreach (var id in files)
|
|
{
|
|
var media = (await library.Db.Media.ListByEntryAsync(id)).Single();
|
|
Assert.AreEqual(MediaState.ToDownload, media.State);
|
|
Assert.AreEqual(0, media.Attempts, "The CDN asked to slow down: not the file's fault.");
|
|
}
|
|
|
|
// Until the pause is over the queue asks the CDN for nothing, however often it is woken.
|
|
queue.Wake();
|
|
Assert.IsNotNull((await queue.RunOnceAsync(CancellationToken.None)).PausedUntil);
|
|
Assert.AreEqual(1, fetcher.Calls);
|
|
|
|
library.Clock.Now = report.PausedUntil!.Value.AddMinutes(1);
|
|
fetcher.Answer = ScriptedFetcher.PhotoByPath;
|
|
var after = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(3, after.Stored, after.ToString());
|
|
Assert.AreEqual(1, breaker.Signals.Count);
|
|
}
|
|
|
|
[TestMethod]
|
|
public void ADownloadStatusIsReportedOncePerWindowButAlwaysRecognized()
|
|
{
|
|
var clock = new VirtualClock();
|
|
var breaker = new RecordingBreaker();
|
|
var signals = new SignalDetector(SignalRules.LoadDefault(), breaker, clock);
|
|
var address = new Uri("https://cdn.esempio/v/t51/abc.jpg?oh=firma");
|
|
Assert.AreEqual(SignalLevel.Throttle, signals.ObserveDownload(address, 429));
|
|
Assert.AreEqual(SignalLevel.Throttle, signals.ObserveDownload(address, 429), "Still a throttle, to stop the caller.");
|
|
Assert.AreEqual(1, breaker.Signals.Count, "Reported once in the dedupe window.");
|
|
Assert.IsNull(signals.ObserveDownload(address, 404));
|
|
Assert.IsNull(signals.ObserveDownload(address, 500));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFailureThatMayPassIsRetriedLaterAndLaterBeforeTheFileFails()
|
|
{
|
|
using var temp = new TempFolder("coda-attesa");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var fetcher = ScriptedFetcher.Network("net::ERR_CONNECTION_RESET");
|
|
var queue = Queue(library, temp, fetcher);
|
|
var start = library.Clock.Now;
|
|
|
|
Assert.AreEqual(1, (await queue.RunOnceAsync(CancellationToken.None)).Retried);
|
|
Assert.AreEqual(0, (await queue.RunOnceAsync(CancellationToken.None)).Total, "Not again at the next pass: it waits.");
|
|
Assert.AreEqual(1, fetcher.Calls);
|
|
|
|
// One pass a minute, as when idle, until the file is settled.
|
|
var passes = 0;
|
|
while ((await library.Db.Media.ListByEntryAsync(entry)).Single().State == MediaState.ToDownload && passes++ < 2000)
|
|
{
|
|
library.Clock.Now = library.Clock.Now.AddMinutes(1);
|
|
await queue.RunOnceAsync(CancellationToken.None);
|
|
}
|
|
|
|
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
Assert.AreEqual(MediaState.Failed, media.State);
|
|
Assert.AreEqual(5, fetcher.Calls, "Five attempts in all.");
|
|
Assert.IsTrue(library.Clock.Now - start >= TimeSpan.FromHours(10.5), $"Spread over {library.Clock.Now - start}, not over a few passes.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task WithoutAConnectionNoAttemptIsCountedAndThePassEnds()
|
|
{
|
|
using var temp = new TempFolder("coda-rete");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entries = new List<long>();
|
|
for (var i = 0; i < 3; i++)
|
|
{
|
|
entries.Add(await AddAsync(library.Db, account, $"p{i}", EntryKind.Story, $"https://cdn.esempio/m/{i}.jpg"));
|
|
}
|
|
|
|
var fetcher = ScriptedFetcher.Network("net::ERR_INTERNET_DISCONNECTED");
|
|
var queue = Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
|
var steps = new List<DownloadProgress>();
|
|
queue.Progress += (_, step) => steps.Add(step);
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, fetcher.Calls, "Every file would fail the same way: the pass ends.");
|
|
Assert.IsNotNull(report.PausedUntil);
|
|
var waiting = steps.Single();
|
|
Assert.AreEqual(DownloadStep.Retried, waiting.Step, "The diagnostic report says why the file is still waiting (issue #39).");
|
|
Assert.AreEqual(DownloadCauses.Connection, waiting.Cause);
|
|
foreach (var id in entries)
|
|
{
|
|
Assert.AreEqual(0, (await library.Db.Media.ListByEntryAsync(id)).Single().Attempts, "The router restarting is not the file's fault.");
|
|
}
|
|
|
|
library.Clock.Now = report.PausedUntil!.Value.AddSeconds(1);
|
|
fetcher.Answer = ScriptedFetcher.PhotoByPath;
|
|
Assert.AreEqual(3, (await queue.RunOnceAsync(CancellationToken.None)).Stored, "Back online: the stories are still there to download.");
|
|
}
|
|
|
|
[TestMethod]
|
|
[Timeout(30_000, CooperativeCancellation = true)]
|
|
public async Task ManyWakeCallsDuringAPassGiveOneMorePassNotOneEach()
|
|
{
|
|
using var temp = new TempFolder("coda-sveglia");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
|
|
// Every pass tries the file again (no wait between attempts here): the passes are the attempts.
|
|
var fetcher = ScriptedFetcher.Status(503);
|
|
var queue = new DownloadQueue(library.Db, library.Ingest, fetcher, new FakeDataBudget(), new IdleClock(), new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), RetryDelays = [], MaxAttempts = 1000 });
|
|
for (var i = 0; i < 50; i++)
|
|
{
|
|
queue.Wake(); // e.g. one per «ricetta.*» event of a long session
|
|
}
|
|
|
|
using var stop = new CancellationTokenSource();
|
|
var running = queue.RunAsync(stop.Token);
|
|
await Task.Delay(TimeSpan.FromMilliseconds(500));
|
|
await stop.CancelAsync();
|
|
await Assert.ThrowsAsync<OperationCanceledException>(() => running);
|
|
Assert.AreEqual(2, fetcher.Calls, "The pass, then one more for all the calls: then it waits.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task FilesWaitingForTheirNextAttemptDoNotHideTheOthers()
|
|
{
|
|
using var temp = new TempFolder("coda-finestra");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
await AddAsync(library.Db, account, "a", EntryKind.Post, "https://cdn.esempio/m/a.jpg");
|
|
await AddAsync(library.Db, account, "b", EntryKind.Post, "https://cdn.esempio/m/b.jpg");
|
|
var good = await AddAsync(library.Db, account, "c", EntryKind.Post, "https://cdn.esempio/m/c.jpg");
|
|
var fetcher = new ScriptedFetcher((uri, destination, ct) => uri!.AbsolutePath.EndsWith("/c.jpg", StringComparison.Ordinal)
|
|
? ScriptedFetcher.Photo(31)(uri, destination, ct)
|
|
: Task.FromResult(new FetchResult(FetchOutcome.HttpError, 0, 503, "prova")));
|
|
var queue = Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), BatchSize = 2 });
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(2, report.Retried, report.ToString());
|
|
Assert.AreEqual(1, report.Stored, "The third file is reached in the same pass.");
|
|
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(good)).Single().State);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnAddressRefusedAgainAfterItsRenewalsEndsFailedWithoutAnotherSinglePost()
|
|
{
|
|
using var temp = new TempFolder("coda-rinnovo");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var renewed = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg", attempts: 4);
|
|
var fresh = await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
|
|
var queue = Queue(library, temp, ScriptedFetcher.Status(403));
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Failed, report.ToString());
|
|
Assert.AreEqual(1, report.Expired);
|
|
var refused = (await library.Db.Media.ListByEntryAsync(renewed)).Single();
|
|
Assert.AreEqual(MediaState.Failed, refused.State);
|
|
StringAssert.Contains(refused.LastError, "rinnovo");
|
|
Assert.AreEqual(MediaState.Expired, (await library.Db.Media.ListByEntryAsync(fresh)).Single().State);
|
|
var tasks = (await library.Db.Tasks.ListPendingAsync(10)).Where(t => t.Kind == EngineTaskKind.SinglePost).ToList();
|
|
CollectionAssert.AreEqual(new[] { fresh.ToString(CultureInfo.InvariantCulture) }, tasks.Select(t => t.Payload).ToArray(), "«Post singolo» only for the first expiry.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnEntryDeletedWhileItsFileDownloadsStaysDeletedAndOutOfTheLibrary()
|
|
{
|
|
using var temp = new TempFolder("coda-eliminato");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var deletion = new LibraryDeletion(library.Db, library.Layout, library.Journal, library.Clock, library.Thumbnails, library.Index);
|
|
|
|
// Alberto deletes the entry («ricorda la scelta») while its file is on its way.
|
|
var fetcher = new ScriptedFetcher(async (uri, destination, ct) =>
|
|
{
|
|
await deletion.DeleteAsync(entry, DeletionMode.Remember, ct);
|
|
return await ScriptedFetcher.Photo(41)(uri, destination, ct);
|
|
});
|
|
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
|
|
|
|
Assert.AreEqual(0, report.Stored, report.ToString());
|
|
Assert.AreEqual(EntryState.Deleted, (await library.Db.Entries.GetAsync(entry))!.State);
|
|
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
Assert.AreEqual(MediaState.Deleted, media.State);
|
|
Assert.IsNull(media.RelativePath);
|
|
var folder = library.Layout.AccountFolder(account);
|
|
Assert.AreEqual(0, Directory.EnumerateFiles(folder, "*.jpg", SearchOption.AllDirectories).Count(), "Nothing entered the library.");
|
|
var lines = (await AccountJournal.ReadAsync(folder)).Lines;
|
|
Assert.AreEqual(JournalOps.Delete, lines[^1].Op, "No «save» line after the «delete» line: a rebuild keeps it deleted.");
|
|
Assert.AreEqual(0, lines.Count(l => l.Op == JournalOps.Save));
|
|
Assert.AreEqual(0, Directory.EnumerateFiles(temp.Combine("dati", "temporanei")).Count());
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AnEntryRemovedWhileItsFileDownloadsLeavesNoOrphanAndTheQueueGoesOn()
|
|
{
|
|
using var temp = new TempFolder("coda-tolto");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var other = await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
|
|
var deletion = new LibraryDeletion(library.Db, library.Layout, library.Journal, library.Clock, library.Thumbnails, library.Index);
|
|
var fetcher = new ScriptedFetcher(async (uri, destination, ct) =>
|
|
{
|
|
if (uri!.AbsolutePath.EndsWith("/1.jpg", StringComparison.Ordinal))
|
|
{
|
|
await deletion.DeleteAsync(entry, DeletionMode.AllowReturn, ct); // «Eliminalo e lascia che torni»
|
|
}
|
|
|
|
return await ScriptedFetcher.PhotoByPath(uri, destination, ct);
|
|
});
|
|
|
|
var report = await Queue(library, temp, fetcher, options: new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 }).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Stored, "The pass goes on with the next file.");
|
|
Assert.IsNull(await library.Db.Entries.GetAsync(entry));
|
|
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(other)).Single().State);
|
|
Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "No orphan file for the removed entry.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFileOfADeletedEntryIsNeverRecordedAsStored()
|
|
{
|
|
using var temp = new TempFolder("coda-registro");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var media = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
await library.Db.Entries.MarkDeletedAsync(entry, library.Clock.Now);
|
|
|
|
var recorded = await library.Db.Media.MarkStoredAsync(media.Id, new StoredFile("post/2026/x_1.jpg", 10, Make.Sha(1)), MediaState.Downloaded, library.Clock.Now);
|
|
Assert.IsFalse(recorded);
|
|
var after = (await library.Db.Media.ListByEntryAsync(entry)).Single();
|
|
Assert.AreEqual(MediaState.Deleted, after.State);
|
|
Assert.IsNull(after.RelativePath);
|
|
Assert.AreEqual(EntryState.Deleted, (await library.Db.Entries.GetAsync(entry))!.State);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TwoIdenticalFilesDownloadedAtTheSameTimeEnterTheLibraryOnce()
|
|
{
|
|
using var temp = new TempFolder("coda-gemelli");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
await AddAsync(library.Db, account, "p2", EntryKind.Post, "https://cdn.esempio/m/2.jpg");
|
|
var bytes = TestFile.PhotoBytes(17, 900, 1100);
|
|
using var together = new Barrier(2);
|
|
var fetcher = new ScriptedFetcher(async (_, destination, ct) =>
|
|
{
|
|
await Task.Run(() => together.SignalAndWait(TimeSpan.FromSeconds(10), ct), ct);
|
|
await destination.WriteAsync(bytes, ct);
|
|
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
|
|
});
|
|
|
|
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Stored, report.ToString());
|
|
Assert.AreEqual(1, report.Duplicates);
|
|
Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "§9.5: one copy of the same bytes per account.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheCleanupOfAnInterruptedRunLeavesTheFilesOnTheirWayAlone()
|
|
{
|
|
using var temp = new TempFolder("coda-pulizia");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var entry = await AddAsync(library.Db, account, "p1", EntryKind.Post, "https://cdn.esempio/m/1.jpg");
|
|
var staging = temp.Combine("dati", "temporanei");
|
|
Directory.CreateDirectory(staging);
|
|
var leftover = Path.Combine(staging, SafeFileWriter.TempPrefix + "vecchio" + SafeFileWriter.TempSuffix);
|
|
await File.WriteAllTextAsync(leftover, "lasciato da un arresto");
|
|
var removed = -1;
|
|
|
|
// The start-up clean-up runs while the first download is being written.
|
|
var bytes = TestFile.PhotoBytes(23, 640, 800);
|
|
var fetcher = new ScriptedFetcher(async (_, destination, ct) =>
|
|
{
|
|
await destination.WriteAsync(bytes.AsMemory(0, 1000), ct);
|
|
removed = SafeFileWriter.CleanupTemporaryFiles(staging, recursive: false);
|
|
await destination.WriteAsync(bytes.AsMemory(1000), ct);
|
|
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
|
|
});
|
|
|
|
var report = await Queue(library, temp, fetcher).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, removed, "Only the file of the previous run.");
|
|
Assert.IsFalse(File.Exists(leftover));
|
|
Assert.AreEqual(1, report.Stored, report.ToString());
|
|
Assert.AreEqual(MediaState.Downloaded, (await library.Db.Media.ListByEntryAsync(entry)).Single().State);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheCleanupLeavesALibraryFileBeingWrittenAlone()
|
|
{
|
|
using var temp = new TempFolder("pulizia-libreria");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var added = await library.Db.Entries.AddAsync(Make.Entry(account.Id, "p1", Make.T0.AddDays(-1)), [Make.Photo()]);
|
|
var folder = library.Layout.AccountFolder(account);
|
|
var removed = -1;
|
|
var content = new InterceptingStream(TestFile.PhotoBytes(29, 640, 800), () => removed = SafeFileWriter.CleanupTemporaryFiles(folder));
|
|
|
|
var result = await library.Ingest.SaveMediaAsync(account, added.Stored.Entry, added.Stored.Media[0], new MediaContent { Content = content, Extension = ".jpg" });
|
|
Assert.AreEqual(0, removed, "The temporary file of the write in progress is not a leftover.");
|
|
Assert.AreEqual(MediaSaveOutcome.Stored, result.Outcome);
|
|
Assert.IsTrue(File.Exists(library.Layout.FullPath(account, result.File!.RelativePath)));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFileOverTheLimitCountsWhatTheBrowserLoadedNotWhatWasWritten()
|
|
{
|
|
using var temp = new TempFolder("coda-limite");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var account = await library.Db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
|
|
await library.Db.Entries.AddAsync(Make.Entry(account.Id, "v1", Make.T0), [Make.Video() with { SourceUrl = "https://cdn.esempio/m/v.mp4", SourceSeenAt = Make.T0 }]);
|
|
var budget = new FakeDataBudget();
|
|
var fetcher = new ScriptedFetcher((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.TooLarge, 0, 200, "troppo grande") { TransferredBytes = 600_000_000 }));
|
|
|
|
var report = await Queue(library, temp, fetcher, budget: budget).RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Skipped);
|
|
Assert.AreEqual(600_000_000, budget.Used, "The daily data budget counts the real transfer.");
|
|
Assert.AreEqual(600_000_000, report.Bytes);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheBrowserFetcherDoesNotReadAFileThatDeclaresMoreThanTheLimit()
|
|
{
|
|
var cdp = new FakeCdpSession(new VirtualClock());
|
|
cdp.Responder = (method, _) => method switch
|
|
{
|
|
"Page.getFrameTree" => new JsonObject { ["frameTree"] = new JsonObject { ["frame"] = new JsonObject { ["id"] = "F1" } } },
|
|
"Network.loadNetworkResource" => new JsonObject
|
|
{
|
|
["resource"] = new JsonObject
|
|
{
|
|
["success"] = true,
|
|
["httpStatusCode"] = 200,
|
|
["stream"] = "S1",
|
|
["headers"] = new JsonObject { ["Content-Type"] = "video/mp4", ["content-length"] = "600000" },
|
|
},
|
|
},
|
|
"IO.read" => new JsonObject { ["data"] = Convert.ToBase64String(new byte[10_000]), ["base64Encoded"] = true, ["eof"] = false },
|
|
_ => null,
|
|
};
|
|
var fetcher = new BrowserFileFetcher(cdp, new HostClassifier(["127.0.0.1:5000"], ["127.0.0.1:5001"]));
|
|
using var destination = new MemoryStream();
|
|
|
|
var result = await fetcher.FetchAsync(new Uri("http://127.0.0.1:5001/grande.mp4"), destination, new FetchOptions(MaxBytes: 100_000), CancellationToken.None);
|
|
Assert.AreEqual(FetchOutcome.TooLarge, result.Outcome);
|
|
Assert.AreEqual(0, destination.Length);
|
|
Assert.AreEqual(600_000, result.TransferredBytes);
|
|
Assert.AreEqual(600_000, result.BudgetBytes);
|
|
Assert.AreEqual(0, cdp.Sent.Count(c => c.Method == "IO.read"), "Nothing more to read: the browser already loaded it.");
|
|
Assert.AreEqual(1, cdp.Sent.Count(c => c.Method == "IO.close"));
|
|
}
|
|
|
|
private static DownloadQueue Queue(TestLibrary library, TempFolder temp, IFileFetcher fetcher, SignalDetector? signals = null, DownloadOptions? options = null, IDataBudget? budget = null) =>
|
|
new(library.Db, library.Ingest, fetcher, budget ?? new FakeDataBudget(), library.Clock, new SeededRandom(1), options ?? new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei") }, signals);
|
|
|
|
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, EntryKind kind, string url, int attempts = 0)
|
|
{
|
|
var entry = Make.Entry(account.Id, id, Make.T0.AddHours(-1)) with
|
|
{
|
|
Kind = kind,
|
|
ExpiresAt = kind == EntryKind.Story ? Make.T0.AddHours(20) : null,
|
|
};
|
|
var added = await db.Entries.AddAsync(entry, [Make.Photo() with { SourceUrl = url, SourceSeenAt = Make.T0, Attempts = attempts }]);
|
|
return added.Stored.Entry.Id;
|
|
}
|
|
|
|
/// <summary>A file fetcher answering from a function that can be changed between passes: no network.</summary>
|
|
private sealed class ScriptedFetcher(Func<Uri?, Stream, CancellationToken, Task<FetchResult>> answer) : IFileFetcher
|
|
{
|
|
private int _calls;
|
|
|
|
public Func<Uri?, Stream, CancellationToken, Task<FetchResult>> Answer { get; set; } = answer;
|
|
|
|
public int Calls => Volatile.Read(ref _calls);
|
|
|
|
public static ScriptedFetcher Status(int status) =>
|
|
new((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.HttpError, 0, status, $"Il server ha risposto {status}.")));
|
|
|
|
public static ScriptedFetcher Network(string error) =>
|
|
new((_, _, _) => Task.FromResult(new FetchResult(FetchOutcome.NetworkError, 0, null, error)));
|
|
|
|
/// <summary>A photo of scene <paramref name="scene"/>, written whole.</summary>
|
|
public static Func<Uri?, Stream, CancellationToken, Task<FetchResult>> Photo(int scene) => async (_, destination, ct) =>
|
|
{
|
|
var bytes = TestFile.PhotoBytes(scene, 320, 400);
|
|
await destination.WriteAsync(bytes, ct);
|
|
return new FetchResult(FetchOutcome.Completed, bytes.Length, 200, null);
|
|
};
|
|
|
|
/// <summary>A photo whose scene depends on the address: different files, different bytes.</summary>
|
|
public static Task<FetchResult> PhotoByPath(Uri? address, Stream destination, CancellationToken ct) =>
|
|
Photo(100 + (address?.AbsolutePath ?? string.Empty).Aggregate(0, (sum, c) => (sum * 31) + c) % 1000)(address, destination, ct);
|
|
|
|
public Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
|
|
{
|
|
Interlocked.Increment(ref _calls);
|
|
return Answer(address, destination, ct);
|
|
}
|
|
}
|
|
|
|
/// <summary>A clock whose short waits (pauses between files) pass at once and whose long ones (idle) never end.</summary>
|
|
private sealed class IdleClock : IClock
|
|
{
|
|
public DateTimeOffset Now => Make.T0;
|
|
|
|
public TimeZoneInfo LocalZone => TestZones.Rome;
|
|
|
|
public Task Delay(TimeSpan time, CancellationToken ct) =>
|
|
time < TimeSpan.FromSeconds(10) ? Task.CompletedTask : Task.Delay(Timeout.Infinite, ct);
|
|
}
|
|
|
|
/// <summary>A content stream that runs <paramref name="midway"/> once, half way through.</summary>
|
|
private sealed class InterceptingStream(byte[] bytes, Action midway) : MemoryStream(bytes)
|
|
{
|
|
private bool _done;
|
|
|
|
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
|
|
{
|
|
var read = await base.ReadAsync(buffer[..Math.Min(buffer.Length, 4096)], cancellationToken);
|
|
if (!_done && Position > Length / 2)
|
|
{
|
|
_done = true;
|
|
midway();
|
|
}
|
|
|
|
return read;
|
|
}
|
|
}
|
|
}
|