Febe non estraeva nulla da Instagram per due motivi. Instagram manda i dati di storie e raccolte (GraphQL) come text/javascript, e la cattura leggeva solo le risposte dichiarate JSON: le scartava tutte. E i file di Instagram (lettori, bersagli) erano scheletri, quindi le ricette si rifiutavano di girare. Alberto ha mandato un HAR del suo browser e una guida. La guida descrive richieste costruite a mano, vietate dalle regole 2-4, e un formato che nel HAR non compare: non si usa. Dal HAR si è letta solo la forma; le risposte utili sono diventate campioni grezzi fuori dal repository, ripuliti e verificati con febe-sonda (tests/Febe.Tests/campioni/instagram). Che cosa cambia: - la cattura legge le risposte text/javascript chieste dalla pagina (mai gli script) e ogni blocco JSON incorporato nel documento della pagina, e avvisa quando un corpo atteso non arriverà; - le risposte che non parlano di contenuti (promozioni, «storia vista») si ignorano invece di valere come formato non riconosciuto; - lettori.instagram.json è confermato, con sole chiavi presenti nei campioni; identificativi delle raccolte nella forma dell'indirizzo; - bersagli.instagram.json ha le pagine viste nel traffico vero; i bersagli restano vuoti (H11): storie, raccolte, Taggati e Reel aspettano lo script tools/bersagli/raccogli-bersagli.js; - le ricette del profilo reggono dati in risposte diverse e prendono la paginazione solo dalla risposta che porta i loro post; l'elenco dei seguiti scarta i profili suggeriti; «Tu» si riconosce anche dal solo identificativo dell'account collegato; - il visualizzatore delle raccolte conta i contenuti (l'indirizzo non cambia), chiede il permesso prima di ogni contenuto e si chiude sempre; - una ricetta che la configurazione non lascia girare è «Da configurare»: non è un guasto, non si sospende, non si pianifica; le sospensioni dovute ai file vecchi si azzerano una volta al cambio dei file; - la barra laterale dice «Collegato» dal cookie di sessione; i «Post singolo» per gli indirizzi scaduti non vengono più annullati; - febe-sonda da-har; la ripulitura tiene latest_reel_media e mette un riempitivo neutro sotto le chiavi simili a segreti (gitleaks). Prove su Windows: 783 superate, 76 saltate (Chromium e Xvfb, nella CI). Nulla è stato provato contro Instagram. Dettagli e decisioni in docs/stato.md e docs/decisioni.md (righe del 25 ottobre). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
429 lines
24 KiB
C#
429 lines
24 KiB
C#
using System.Collections.Concurrent;
|
|
using Febe.Browser.Targets;
|
|
using Febe.Core.Abstractions;
|
|
using Febe.Core.Collection;
|
|
using Febe.Core.Collection.Downloads;
|
|
using Febe.Core.Collection.Readers;
|
|
using Febe.Core.Collection.Recipes;
|
|
using Febe.Core.Domain;
|
|
using Febe.Core.Engine;
|
|
using Febe.Core.Storage;
|
|
using Febe.Tests.Browser.TestSupport;
|
|
using Febe.Tests.Collection.Support;
|
|
using Febe.Tests.Support;
|
|
|
|
namespace Febe.Tests.Collection;
|
|
|
|
/// <summary>Site map, recipe guards, the writer and the download queue, without a browser.</summary>
|
|
[TestClass]
|
|
public sealed class CollectionUnitTests
|
|
{
|
|
[TestMethod]
|
|
public void TheTestSiteMapBuildsAndReadsAddresses()
|
|
{
|
|
var map = SiteMap.LoadTestSite();
|
|
Assert.AreEqual("/profilo/marta.prova/", map.BuildPath(SitePages.Profile, new Dictionary<string, string> { ["username"] = "marta.prova" }));
|
|
Assert.AreEqual("/profilo/a%20b/?dopo=c%2612", map.BuildPath(SitePages.ProfileFrom, new Dictionary<string, string> { ["username"] = "a b", ["cursor"] = "c&12" }));
|
|
var story = map.Match(SitePages.StoryViewer, "/storie/marta.prova/7300105520/");
|
|
Assert.AreEqual("marta.prova", story!["username"]);
|
|
Assert.AreEqual("7300105520", story["id"]);
|
|
Assert.IsNull(map.Match(SitePages.StoryViewer, "/profilo/marta.prova/"));
|
|
Assert.IsNotNull(map.Match(SitePages.Home, "/"));
|
|
Assert.AreEqual(0, map.Missing(SitePages.All, SiteTargets.All).Count);
|
|
Assert.IsTrue(File.Exists(Path.Combine(AppContext.BaseDirectory, "config", "bersagli.prova.json")), "Editable without recompiling.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public void TheInstagramMapHasThePagesSeenInRealTrafficAndNoTargetYet()
|
|
{
|
|
var map = SiteMap.LoadInstagram();
|
|
Assert.AreEqual("instagram", map.Site);
|
|
|
|
// Pages: only those with evidence in the HAR of 21 October 2026 (Instagram's own route templates and the
|
|
// Referer of the page's requests). The others have none and stay empty.
|
|
string[] filled = [SitePages.Home, SitePages.Profile, SitePages.Post, SitePages.StoryViewer, SitePages.HighlightViewer];
|
|
string[] empty = [SitePages.ProfileFrom, SitePages.ReelsFrom, SitePages.TaggedFrom, SitePages.Following, SitePages.FollowingFrom];
|
|
CollectionAssert.AreEquivalent(SitePages.All.ToArray(), filled.Concat(empty).ToArray(), "Every page is decided.");
|
|
Assert.IsTrue(filled.All(map.HasPage));
|
|
CollectionAssert.AreEquivalent(empty.Select(p => $"pagina «{p}»").ToArray(), map.Missing(SitePages.All, []).ToArray());
|
|
|
|
Assert.AreEqual("/", map.BuildPath(SitePages.Home));
|
|
Assert.AreEqual("/mario.rossi/", map.BuildPath(SitePages.Profile, new Dictionary<string, string> { ["username"] = "mario.rossi" }));
|
|
Assert.AreEqual("/p/Abc_12-xYz9/", map.BuildPath(SitePages.Post, new Dictionary<string, string> { ["code"] = "Abc_12-xYz9" }));
|
|
Assert.IsNotNull(map.Match(SitePages.Home, "/"));
|
|
Assert.IsNull(map.Match(SitePages.Home, "/mario.rossi/"));
|
|
Assert.AreEqual("mario.rossi", map.Match(SitePages.Profile, "/mario.rossi/")!["username"]);
|
|
|
|
// Viewers: matched only, never opened. The story viewer carries the account and the story shown.
|
|
var story = map.Match(SitePages.StoryViewer, "/stories/mario/3456789012345678901/");
|
|
Assert.AreEqual("mario", story!["username"]);
|
|
Assert.AreEqual("3456789012345678901", story["id"]);
|
|
Assert.IsNull(map.Match(SitePages.StoryViewer, "/stories/mario/"), "Without the story id the shape does not match (to be confirmed live).");
|
|
Assert.IsNull(map.Match(SitePages.HighlightViewer, "/stories/mario/3456789012345678901/"));
|
|
|
|
// The highlight viewer: the 17 digits of the collection and no item id (the address stays the same while watching).
|
|
var highlight = map.Match(SitePages.HighlightViewer, "/stories/highlights/17841400000000000/");
|
|
CollectionAssert.AreEquivalent(new[] { "highlight" }, highlight!.Keys.ToArray());
|
|
Assert.AreEqual("17841400000000000", highlight["highlight"]);
|
|
|
|
// Overlap: a highlight address also has the story viewer's shape, with «highlights» as user name. The story
|
|
// recipe then sees «another account» and closes; the highlight recipe reads only its own page.
|
|
Assert.AreEqual("highlights", map.Match(SitePages.StoryViewer, "/stories/highlights/17841400000000000/")!["username"]);
|
|
|
|
// Targets: none has evidence yet (the HAR had no documents and no element attributes).
|
|
var missingTargets = map.Missing([], SiteTargets.All);
|
|
Assert.AreEqual(SiteTargets.All.Count, missingTargets.Count);
|
|
Assert.IsTrue(missingTargets.All(m => m.EndsWith("(vuoto)", StringComparison.Ordinal)), "Every target is listed, still empty.");
|
|
|
|
// The browser layer reads the same file: every target is a placeholder and is never looked up in the page.
|
|
var targets = TargetMap.LoadEmbedded("bersagli.instagram.json");
|
|
CollectionAssert.AreEquivalent(SiteTargets.All.ToArray(), targets.Targets.Keys.ToArray());
|
|
Assert.IsTrue(targets.Targets.Values.All(t => !t.IsConfigured && t.Hypothesis == "H11"));
|
|
var cdp = new FakeCdpSession(new VirtualClock());
|
|
var lookup = new TargetResolver(cdp, targets).ResolveAsync(SiteTargets.ViewerClose).GetAwaiter().GetResult();
|
|
Assert.AreEqual(TargetStatus.NotConfigured, lookup.Status);
|
|
Assert.AreEqual(0, cdp.Sent.Count, "A placeholder never reaches the page.");
|
|
StringAssert.Contains(lookup.Message, "H11");
|
|
|
|
// The copy next to the executable is the same map.
|
|
var copy = SiteMap.Load(Path.Combine(AppContext.BaseDirectory, "config", "bersagli.instagram.json"));
|
|
CollectionAssert.AreEquivalent(map.Missing(SitePages.All, SiteTargets.All).ToArray(), copy.Missing(SitePages.All, SiteTargets.All).ToArray());
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task OnInstagramOnlyRecipesThatNeedNoTargetGetPastTheMap()
|
|
{
|
|
using var temp = new TempFolder("ricette");
|
|
await using var db = await TestDatabase.OpenAsync(temp);
|
|
var clock = new VirtualClock();
|
|
var governor = new PermissiveGovernor();
|
|
var session = new TestSession((await db.Sessions.StartAsync(clock.Now)).Id);
|
|
|
|
// The Instagram vocabulary is confirmed on real samples. To see which recipes get past the map without letting
|
|
// any of them reach the page, the runs use a copy marked as a hypothesis: the readers' gate comes right after
|
|
// the map's and stops them there.
|
|
Assert.IsTrue(ReaderVocabulary.LoadInstagram().IsUsable, ReaderVocabulary.LoadInstagram().UnusableReason);
|
|
var stopper = ReaderVocabulary.LoadInstagram() with { Status = VocabularyStatus.Hypothesis };
|
|
Task<RecipeOutcome> RunAsync(RecipeKind kind) => new RecipeRunner(new RecipeEnvironment
|
|
{
|
|
Driver = new UntouchableDriver(),
|
|
Capture = new CaptureCollector(new ShapeReader(stopper), clock),
|
|
Governor = governor,
|
|
Signals = new SignalDetector(SignalRules.LoadDefault(), new RecordingBreaker(), clock),
|
|
Storage = db,
|
|
SiteMap = SiteMap.LoadInstagram(),
|
|
Clock = clock,
|
|
Random = new SeededRandom(1),
|
|
}).RunAsync(new RecipeRequest(kind, 1), session, CancellationToken.None);
|
|
|
|
static void Refused(RecipeKind kind, RecipeOutcome outcome, string missing)
|
|
{
|
|
Assert.AreEqual(RecipeStatus.NotConfigured, outcome.Status, kind.ToString());
|
|
StringAssert.Contains(outcome.Message, "mappa dei bersagli non è ancora compilata");
|
|
StringAssert.Contains(outcome.Message, missing, kind.ToString());
|
|
}
|
|
|
|
static void Passed(RecipeKind kind, RecipeOutcome outcome)
|
|
{
|
|
Assert.AreEqual(RecipeStatus.NotConfigured, outcome.Status, $"{kind}: {outcome.Message}");
|
|
StringAssert.Contains(outcome.Message, "ipotesi", kind.ToString());
|
|
}
|
|
|
|
// Recipes that click something: every target is still empty (H11), while their pages are all there.
|
|
foreach (var kind in new[] { RecipeKind.AccountStories, RecipeKind.Highlight, RecipeKind.Tagged, RecipeKind.Reels })
|
|
{
|
|
var outcome = await RunAsync(kind);
|
|
Refused(kind, outcome, "(vuoto)");
|
|
Assert.IsFalse(outcome.Message!.Contains("pagina «", StringComparison.Ordinal), $"{kind}: {outcome.Message}");
|
|
}
|
|
|
|
// Recipes that only open pages.
|
|
Passed(RecipeKind.FollowingSweep, await RunAsync(RecipeKind.FollowingSweep));
|
|
Passed(RecipeKind.SinglePost, await RunAsync(RecipeKind.SinglePost));
|
|
|
|
// The profile check and the backfill need no target: the «not found» marker is looked up only when configured.
|
|
foreach (var kind in new[] { RecipeKind.ProfileCheck, RecipeKind.Backfill })
|
|
{
|
|
Assert.IsFalse(RecipeRunner.Requirements(kind).Targets.Contains(SiteTargets.ProfileNotFound), kind.ToString());
|
|
Passed(kind, await RunAsync(kind));
|
|
}
|
|
|
|
// No evidence of the address of the following list.
|
|
Refused(RecipeKind.FollowingList, await RunAsync(RecipeKind.FollowingList), $"pagina «{SitePages.Following}»");
|
|
|
|
Assert.AreEqual(0, governor.Requests.Count, "No permit asked, no gesture, no page.");
|
|
var events = await db.Events.ListAsync(new EventQuery());
|
|
Assert.IsTrue(events.Any(e => e.Code == "mappa-incompleta"));
|
|
Assert.IsTrue(events.Any(e => e.Code == "lettori-incompleti"));
|
|
}
|
|
|
|
[TestMethod]
|
|
public void StoryTimingsFollowSection77()
|
|
{
|
|
var random = new SeededRandom(3);
|
|
for (var i = 0; i < 500; i++)
|
|
{
|
|
var photo = StoryTimings.Watch(random, null, 1.0);
|
|
Assert.IsTrue(photo >= TimeSpan.FromSeconds(1.8) && photo <= TimeSpan.FromSeconds(5), photo.ToString());
|
|
var video = StoryTimings.Watch(random, TimeSpan.FromSeconds(10), 1.0);
|
|
Assert.IsTrue(video >= TimeSpan.FromSeconds(4) && video <= TimeSpan.FromSeconds(10), video.ToString());
|
|
var longVideo = StoryTimings.Watch(random, TimeSpan.FromSeconds(60), 1.0);
|
|
Assert.IsTrue(longVideo <= TimeSpan.FromSeconds(15), "At most 15 s.");
|
|
var between = StoryTimings.BetweenAccounts(random, 1.0);
|
|
Assert.IsTrue(between >= TimeSpan.FromSeconds(1) && between <= TimeSpan.FromSeconds(4));
|
|
}
|
|
|
|
Assert.AreEqual(2 * StoryTimings.Watch(new SeededRandom(9), null, 1.0).Ticks, StoryTimings.Watch(new SeededRandom(9), null, 2.0).Ticks, 1, "Mood and pace multiply the waits.");
|
|
Assert.AreEqual("48.212", ItalianText.Number(48212));
|
|
Assert.AreEqual("1.000.000", ItalianText.Number(1_000_000));
|
|
Assert.AreEqual("96", ItalianText.Number(96));
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheWriterRecognizesImportedEntriesAndAppliesTheAccountSettings()
|
|
{
|
|
using var temp = new TempFolder("scrittore");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var db = library.Db;
|
|
var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with
|
|
{
|
|
Limits = new AccountLimits { Depth = HistoryDepth.FromDate, FromDate = Make.T0.AddDays(-10), StoryVideos = false },
|
|
});
|
|
var writer = new CollectionWriter(db, library.Clock);
|
|
var imported = await db.Entries.AddAsync(new Entry
|
|
{
|
|
AccountId = account.Id,
|
|
Kind = EntryKind.Post,
|
|
PublishedAt = Make.T0.AddDays(-1),
|
|
Provenance = Provenance.Imported,
|
|
DiscoveredAt = Make.T0,
|
|
}, []);
|
|
|
|
var reader = new ShapeReader(ReaderVocabulary.LoadTestSite());
|
|
ContentItem Item(string id, DateTimeOffset at, bool video = false) => reader.Read($$"""
|
|
{"numero":"{{id}}","sigla":"S{{id}}","pubblicato":"{{at.UtcDateTime:yyyy-MM-ddTHH:mm:ssZ}}","testo":"didascalia","autore":{"numero":"1001","nome":"marta.prova"},
|
|
"{{(video ? "filmati" : "foto")}}":[{"indirizzo":"https://cdn.esempio/{{id}}.{{(video ? "mp4" : "jpg")}}","larghezza":10,"altezza":10,"audio":true}]}
|
|
""").Contents.Single();
|
|
|
|
var enriched = await writer.SaveAsync(account, Item("500", Make.T0.AddDays(-1).AddSeconds(1)), EntryKind.Post, null, CancellationToken.None);
|
|
Assert.AreEqual(SaveOutcome.Enriched, enriched.Outcome, "Same account, kind and time within two seconds (§9.5).");
|
|
var stored = await db.Entries.GetAsync(imported.Stored.Entry.Id);
|
|
Assert.AreEqual("500", stored!.InstagramId);
|
|
Assert.AreEqual("S500", stored.Code);
|
|
|
|
Assert.AreEqual(SaveOutcome.Excluded, (await writer.SaveAsync(account, Item("501", Make.T0.AddDays(-30)), EntryKind.Post, null, CancellationToken.None)).Outcome, "Older than «da una data».");
|
|
Assert.AreEqual(SaveOutcome.Excluded, (await writer.SaveAsync(account, Item("502", Make.T0), EntryKind.Tagged, null, CancellationToken.None)).Outcome, "«Post in cui è taggato» is off.");
|
|
Assert.AreEqual(SaveOutcome.Created, (await writer.SaveAsync(account, Item("503", Make.T0), EntryKind.Post, null, CancellationToken.None)).Outcome);
|
|
Assert.AreEqual(SaveOutcome.Known, (await writer.SaveAsync(account, Item("503", Make.T0), EntryKind.Post, null, CancellationToken.None)).Outcome);
|
|
|
|
var story = await writer.SaveAsync(account, Item("504", Make.T0, video: true), EntryKind.Story, null, CancellationToken.None);
|
|
var video = (await db.Media.ListByEntryAsync(story.Entry!.Id)).Single();
|
|
Assert.AreEqual(MediaState.Skipped, video.State, "«Video delle storie» is off: never downloaded.");
|
|
Assert.IsNull(video.SourceUrl);
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheQueueRecoversExpiredAddressesSkipsTooLargeVideosAndPutsStoriesFirst()
|
|
{
|
|
using var temp = new TempFolder("coda");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var db = library.Db;
|
|
var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001") with { Limits = new AccountLimits { MaxVideoBytes = 1000 } });
|
|
var post = await AddAsync(db, account, "p1", EntryKind.Post, Make.Photo(), "https://cdn.esempio/scaduto.jpg");
|
|
var story = await AddAsync(db, account, "s1", EntryKind.Story, Make.Photo(), "https://cdn.esempio/storia-scaduta.jpg");
|
|
var video = await AddAsync(db, account, "v1", EntryKind.Post, Make.Video(), "https://cdn.esempio/grande.mp4");
|
|
var fine = await AddAsync(db, account, "f1", EntryKind.Story, Make.Photo(), "https://cdn.esempio/buono.jpg");
|
|
var fetcher = new FakeFetcher(uri => uri.AbsolutePath switch
|
|
{
|
|
"/scaduto.jpg" or "/storia-scaduta.jpg" => (FetchOutcome.HttpError, 403, null),
|
|
"/grande.mp4" => (FetchOutcome.Completed, 200, new byte[5000]),
|
|
_ => (FetchOutcome.Completed, 200, TestFile.PhotoBytes(3, 64, 80)),
|
|
});
|
|
var queue = new DownloadQueue(db, library.Ingest, fetcher, new FakeDataBudget(), library.Clock, new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
|
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Stored, report.ToString());
|
|
Assert.AreEqual(2, report.Expired);
|
|
Assert.AreEqual(1, report.Skipped);
|
|
CollectionAssert.AreEqual(new[] { "/storia-scaduta.jpg", "/buono.jpg" }, fetcher.Order.Take(2).ToArray(), "Stories first: they expire.");
|
|
|
|
Assert.AreEqual(MediaState.Expired, (await db.Media.ListByEntryAsync(post)).Single().State);
|
|
Assert.AreEqual(MediaState.Failed, (await db.Media.ListByEntryAsync(story)).Single().State, "No recovery for stories.");
|
|
var tasks = await db.Tasks.ListPendingAsync(10);
|
|
Assert.AreEqual(post.ToString(System.Globalization.CultureInfo.InvariantCulture), tasks.Single(t => t.Kind == EngineTaskKind.SinglePost).Payload);
|
|
var big = (await db.Media.ListByEntryAsync(video)).Single();
|
|
Assert.AreEqual(MediaState.Skipped, big.State);
|
|
StringAssert.Contains(big.LastError, "limite");
|
|
Assert.AreEqual(MediaState.Downloaded, (await db.Media.ListByEntryAsync(fine)).Single().State);
|
|
Assert.AreEqual(0, Directory.EnumerateFiles(temp.Combine("dati", "temporanei")).Count());
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task AFileIdenticalToAnImportedOneIsDiscardedAndTheImportedEntryTakesTheData()
|
|
{
|
|
using var temp = new TempFolder("doppioni");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var db = library.Db;
|
|
var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
var bytes = TestFile.PhotoBytes(11, 120, 150);
|
|
var imported = await library.SaveAsync(account, new Entry
|
|
{
|
|
AccountId = account.Id,
|
|
Kind = EntryKind.Post,
|
|
PublishedAt = Make.T0.AddDays(-3),
|
|
Provenance = Provenance.Imported,
|
|
DiscoveredAt = Make.T0,
|
|
}, [new TestFile(Make.Photo(), bytes, ".jpg")]);
|
|
var downloaded = await AddAsync(db, account, "777", EntryKind.Post, Make.Photo(), "https://cdn.esempio/stesso.jpg", caption: "dal sito");
|
|
|
|
var queue = new DownloadQueue(db, library.Ingest, new FakeFetcher(_ => (FetchOutcome.Completed, 200, bytes)), new FakeDataBudget(), library.Clock, new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei") });
|
|
var report = await queue.RunOnceAsync(CancellationToken.None);
|
|
Assert.AreEqual(1, report.Duplicates, report.ToString());
|
|
Assert.IsNull(await db.Entries.GetAsync(downloaded), "The new record goes away.");
|
|
var merged = await db.Entries.GetAsync(imported.Entry.Id);
|
|
Assert.AreEqual("777", merged!.InstagramId);
|
|
Assert.AreEqual("dal sito", merged.Caption);
|
|
Assert.AreEqual(1, Directory.EnumerateFiles(library.Layout.AccountFolder(account), "*.jpg", SearchOption.AllDirectories).Count(), "One file in the library.");
|
|
}
|
|
|
|
[TestMethod]
|
|
public async Task TheQueueStopsAtTheDataBudgetAndNeverRunsMoreThanTwoAtOnce()
|
|
{
|
|
using var temp = new TempFolder("tetto");
|
|
await using var library = await TestLibrary.CreateAsync(temp);
|
|
var db = library.Db;
|
|
var account = await db.Accounts.AddAsync(Make.Account("marta.prova", "1001"));
|
|
for (var i = 0; i < 8; i++)
|
|
{
|
|
await AddAsync(db, account, $"p{i}", EntryKind.Post, Make.Photo(), $"https://cdn.esempio/{i}.jpg");
|
|
}
|
|
|
|
var fetcher = new FakeFetcher(uri => (FetchOutcome.Completed, 200, TestFile.PhotoBytes(int.Parse(uri.AbsolutePath[1..^4], System.Globalization.CultureInfo.InvariantCulture) + 20, 64, 80)))
|
|
{
|
|
Latency = TimeSpan.FromMilliseconds(30),
|
|
};
|
|
var queue = new DownloadQueue(db, library.Ingest, fetcher, new FakeDataBudget(), library.Clock, new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei") });
|
|
Assert.AreEqual(8, (await queue.RunOnceAsync(CancellationToken.None)).Stored);
|
|
Assert.AreEqual(2, fetcher.MaxConcurrent, "At most two at the same time (§6.5).");
|
|
|
|
for (var i = 8; i < 12; i++)
|
|
{
|
|
await AddAsync(db, account, $"p{i}", EntryKind.Post, Make.Photo(), $"https://cdn.esempio/{i}.jpg");
|
|
}
|
|
|
|
var budget = new FakeDataBudget(bytesPerDay: 1);
|
|
var limited = new DownloadQueue(db, library.Ingest, fetcher, budget, library.Clock, new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Combine("dati", "temporanei"), MaxConcurrent = 1 });
|
|
var report = await limited.RunOnceAsync(CancellationToken.None);
|
|
Assert.IsTrue(report.BudgetReached);
|
|
Assert.AreEqual(1, report.Stored, "The budget is checked before each file.");
|
|
Assert.AreEqual(3, await db.Media.CountPendingDownloadsAsync());
|
|
Assert.ThrowsExactly<ArgumentOutOfRangeException>(() => new DownloadQueue(db, library.Ingest, fetcher, budget, library.Clock, new SeededRandom(1),
|
|
new DownloadOptions { StagingFolder = temp.Path, MaxConcurrent = 3 }));
|
|
}
|
|
|
|
private static async Task<long> AddAsync(IFebeStorage db, Account account, string id, EntryKind kind, Media media, string url, string? caption = null)
|
|
{
|
|
var entry = new Entry
|
|
{
|
|
AccountId = account.Id,
|
|
Kind = kind,
|
|
InstagramId = id,
|
|
Code = "C" + id,
|
|
PublishedAt = Make.T0.AddHours(-id.Length),
|
|
Caption = caption,
|
|
DiscoveredAt = Make.T0,
|
|
ExpiresAt = kind == EntryKind.Story ? Make.T0.AddHours(20) : null,
|
|
};
|
|
var added = await db.Entries.AddAsync(entry, [media with { SourceUrl = url, SourceSeenAt = Make.T0 }]);
|
|
return added.Stored.Entry.Id;
|
|
}
|
|
|
|
/// <summary>A file fetcher answering from a table: no network at all.</summary>
|
|
private sealed class FakeFetcher(Func<Uri, (FetchOutcome Outcome, int? Status, byte[]? Bytes)> answer) : IFileFetcher
|
|
{
|
|
private int _current;
|
|
private int _max;
|
|
|
|
public ConcurrentQueue<string> Calls { get; } = new();
|
|
|
|
public IReadOnlyList<string> Order => [.. Calls];
|
|
|
|
public TimeSpan Latency { get; init; }
|
|
|
|
public int MaxConcurrent => Volatile.Read(ref _max);
|
|
|
|
public async Task<FetchResult> FetchAsync(Uri address, Stream destination, FetchOptions? options, CancellationToken ct)
|
|
{
|
|
Calls.Enqueue(address.AbsolutePath);
|
|
var now = Interlocked.Increment(ref _current);
|
|
int seen;
|
|
while ((seen = Volatile.Read(ref _max)) < now && Interlocked.CompareExchange(ref _max, now, seen) != seen)
|
|
{
|
|
}
|
|
|
|
try
|
|
{
|
|
if (Latency > TimeSpan.Zero)
|
|
{
|
|
await Task.Delay(Latency, ct);
|
|
}
|
|
|
|
var (outcome, status, bytes) = answer(address);
|
|
if (bytes is null)
|
|
{
|
|
return new FetchResult(outcome, 0, status, "errore di prova");
|
|
}
|
|
|
|
if (options?.MaxBytes is { } max && bytes.Length > max)
|
|
{
|
|
await destination.WriteAsync(bytes.AsMemory(0, (int)max), ct);
|
|
return new FetchResult(FetchOutcome.TooLarge, max, status, "troppo grande");
|
|
}
|
|
|
|
await destination.WriteAsync(bytes, ct);
|
|
return new FetchResult(outcome, bytes.Length, status, null);
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Decrement(ref _current);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>A driver that fails the test if a recipe touches it.</summary>
|
|
private sealed class UntouchableDriver : IPageDriver
|
|
{
|
|
public double WaitMultiplier => 1;
|
|
|
|
public void BeginSession(double mood)
|
|
{
|
|
}
|
|
|
|
public Task<PageLoad> OpenAsync(string path, CancellationToken ct) => throw new AssertFailedException("La ricetta ha aperto una pagina.");
|
|
|
|
public Task<string> CurrentPathAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha letto la pagina.");
|
|
|
|
public Task<bool> IsOnSiteAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha letto la pagina.");
|
|
|
|
public Task<TargetCheck> FindAsync(string target, IReadOnlyDictionary<string, string>? parameters, CancellationToken ct) => throw new AssertFailedException("La ricetta ha cercato un bersaglio.");
|
|
|
|
public Task<TargetCheck> ClickAsync(string target, IReadOnlyDictionary<string, string>? parameters, CancellationToken ct) => throw new AssertFailedException("La ricetta ha cliccato.");
|
|
|
|
public Task ScrollBurstAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha scorso la pagina.");
|
|
|
|
public Task WaitAfterLoadAsync(CancellationToken ct) => Task.CompletedTask;
|
|
|
|
public Task WaitAsync(TimeSpan duration, CancellationToken ct) => Task.CompletedTask;
|
|
|
|
public Task<bool> MaybeDistractionAsync(CancellationToken ct) => Task.FromResult(false);
|
|
|
|
public Task<bool?> HasSessionCookieAsync(CancellationToken ct) => throw new AssertFailedException("La ricetta ha controllato il cookie.");
|
|
|
|
public Task RestAsync(CancellationToken ct) => Task.CompletedTask;
|
|
}
|
|
}
|