using System.Globalization; using System.Text; using Febe.Core.Abstractions; using Febe.Core.Domain; using Febe.Core.Engine; using Febe.Core.Storage; namespace Febe.Core.Simulation; /// Numbers of one simulated day. public sealed record SimulatedDay( DateOnly Day, int Sessions, double ActiveMinutes, IReadOnlyDictionary Actions, IReadOnlyDictionary WorstHour, int Throttles, int Overruns); /// Numbers of one budget category over the whole simulation. public sealed record SimulatedCategory(ActionKind Kind, double MeanPerDay, long MaxPerDay, long WorstHour, long? HourlyLimit, long DailyLimit); /// Mean and maximum of a delay, with how many cases. public sealed record DelayStats(int Count, TimeSpan Mean, TimeSpan Max) { public static DelayStats From(IEnumerable delays) { var list = delays.ToList(); return list.Count == 0 ? new DelayStats(0, TimeSpan.Zero, TimeSpan.Zero) : new DelayStats(list.Count, TimeSpan.FromTicks((long)list.Average(d => d.Ticks)), list.Max()); } } /// /// The report of the simulation (spec §14.3): sessions per day; actions per category per day and in the worst hour; /// mean and maximum time between publication and discovery; days for the 2,000-post backfill; budget overruns (must be /// zero). Overruns, actions outside the window and pause-floor violations are recomputed here from the database, /// independently of the Governor. /// public sealed class SimulationReport { private static readonly ActionKind[] Categories = Enum.GetValues(); private static readonly TimeSpan Hour = TimeSpan.FromMinutes(60); public required CautionProfileKind Profile { get; init; } /// Whether the simulated recipes went by the pages or by requests. public SimulatedCollection Collection { get; init; } public required int Seed { get; init; } public required DateTimeOffset Start { get; init; } public required DateTimeOffset End { get; init; } public required TimeZoneInfo Zone { get; init; } public required int Accounts { get; init; } public required int FollowedAccounts { get; init; } public required IReadOnlyList Days { get; init; } public required IReadOnlyList CategoryTotals { get; init; } public required int SessionCount { get; init; } public required double SessionsPerDayMean { get; init; } public required int SessionsPerDayMin { get; init; } public required int SessionsPerDayMax { get; init; } public required TimeSpan MeanSessionLength { get; init; } /// Coefficient of variation of the intervals between session starts (whole run). public required double SessionStartCv { get; init; } /// Same, only for intervals inside one day (nights excluded). public required double SessionStartCvWithinDay { get; init; } public required TimeSpan ShortestPause { get; init; } public required int PauseFloorViolations { get; init; } public required int ActionsOutsideWindow { get; init; } public required DelayStats FollowedPosts { get; init; } public required DelayStats NotFollowedPosts { get; init; } public required DelayStats Stories { get; init; } public required int PublishedPosts { get; init; } public required int UndiscoveredPosts { get; init; } public required int PublishedStories { get; init; } public required int MissedStories { get; init; } public required int BackfillPosts { get; init; } public required int BackfillStored { get; init; } public DateTimeOffset? BackfillFinishedAt { get; init; } public required IReadOnlyList Throttles { get; init; } /// Times Febe proposed a more prudent profile (three throttles in seven days). public required int ProposalsOfMorePrudence { get; init; } /// Overruns of the budgets in force (gradual start and halving applied): must be zero. public required int Overruns { get; init; } /// Overruns of the profile's full budgets: must be zero too. public required int OverrunsOfFullBudgets { get; init; } public required IReadOnlyList OverrunDetails { get; init; } public required long VirtualEvents { get; init; } /// What the fake world saw (for tests and closer looks). public required WorldStats World { get; init; } /// The accounts of the fake world. public required IReadOnlyList WorldAccounts { get; init; } public required TimeSpan RealDuration { get; init; } public double? BackfillDays => BackfillFinishedAt is { } done ? (done - Start).TotalDays : null; internal static async Task BuildAsync( IFebeStorage storage, SimulatedWorld world, EngineServices services, SimulationOptions options, DateTimeOffset start, DateTimeOffset end, long virtualEvents, TimeSpan realDuration, CancellationToken ct) { var zone = services.Clock.LocalZone; var profile = services.Runtime.Profile; var sessions = (await storage.Sessions.ListAsync(start, end + TimeSpan.FromDays(1), ct).ConfigureAwait(false)) .Where(s => s.StartedAt < end) .ToList(); var actions = await storage.ActionLog.ListAsync(start, end + TimeSpan.FromDays(1), ct).ConfigureAwait(false); var incidents = await storage.Incidents.ListAsync(start, end + TimeSpan.FromDays(1), ct).ConfigureAwait(false); var throttles = incidents.Where(i => i.Level == SignalLevel.Throttle).ToList(); var seedText = await storage.Settings.GetAsync(EngineSettingKeys.WindowSeed, ct).ConfigureAwait(false); var seed = ulong.TryParse(seedText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var s) ? s : 0; var rampStart = services.Runtime.RampStart; // Halving applies after the throttle: the action that met the signal was granted before it. DateTimeOffset? HalvedUntil(DateTimeOffset t) => throttles .Where(i => i.At < t && i.StoppedUntil is not null) .Select(i => (DateTimeOffset?)(i.StoppedUntil!.Value + CircuitBreaker.HalvedPeriod)) .Where(u => u > t) .DefaultIfEmpty(null) .Max(); // Overruns, recomputed from action_log: at every action the rolling hour and the local day so far. var overruns = 0; var overrunsFull = 0; var details = new List(); var overrunsByDay = new Dictionary(); var worstHourByDay = new Dictionary<(DateOnly, ActionKind), long>(); var worstHour = new Dictionary(); foreach (var kind in Categories) { var list = actions.Where(a => a.Category == kind).OrderBy(a => a.At).ThenBy(a => a.Id).ToList(); var windowStart = 0; long windowSum = 0; long daySum = 0; DateOnly? currentDay = null; for (var i = 0; i < list.Count; i++) { var action = list[i]; windowSum += action.Amount; while (list[windowStart].At <= action.At - Hour) { windowSum -= list[windowStart].Amount; windowStart++; } var day = LocalCalendar.Today(action.At, zone); if (day != currentDay) { currentDay = day; daySum = 0; } daySum += action.Amount; worstHour[kind] = Math.Max(worstHour.GetValueOrDefault(kind), windowSum); worstHourByDay[(day, kind)] = Math.Max(worstHourByDay.GetValueOrDefault((day, kind)), windowSum); var limits = BudgetLimits.Compute(profile, rampStart, HalvedUntil(action.At), action.At); var over = kind == ActionKind.DataDownload ? daySum > limits.DataPerDayBytes : windowSum > limits.Budgets[kind].PerHour || daySum > limits.Budgets[kind].PerDay; var overFull = kind == ActionKind.DataDownload ? daySum > profile.DataPerDayBytes : windowSum > profile.Budgets[kind].PerHour || daySum > profile.Budgets[kind].PerDay; if (over) { overruns++; overrunsByDay[day] = overrunsByDay.GetValueOrDefault(day) + 1; if (details.Count < 20) { details.Add(string.Create( CultureInfo.InvariantCulture, $"{ItalianText.Category(kind)} {LocalCalendar.ToLocal(action.At, zone):yyyy-MM-dd HH:mm:ss}: ora {windowSum}, giorno {daySum}")); } } if (overFull) { overrunsFull++; } } } // Actions outside the active window (recomputed with the persisted seed of the daily shifts). var outside = 0; var windows = new Dictionary(); ActiveWindow? WindowOf(DateOnly day) { if (!windows.TryGetValue(day, out var w)) { w = profile.ActiveDays.Contains(day.DayOfWeek) ? DayPlanner.Compute(day, profile, seed, zone) : null; windows[day] = w; } return w; } foreach (var action in actions.Where(a => a.Category != ActionKind.DataDownload)) { var today = LocalCalendar.Today(action.At, zone); var inside = (WindowOf(today)?.Contains(action.At) ?? false) || (WindowOf(today.AddDays(-1))?.Contains(action.At) ?? false); if (!inside) { outside++; } } // Pauses and irregularity of the session starts. var ordered = sessions.OrderBy(x => x.StartedAt).ToList(); var shortest = TimeSpan.MaxValue; var floorViolations = 0; for (var i = 1; i < ordered.Count; i++) { if (ordered[i - 1].EndedAt is { } previousEnd) { var pause = ordered[i].StartedAt - previousEnd; shortest = pause < shortest ? pause : shortest; if (pause < profile.PauseMin) { floorViolations++; } } } var intervals = new List(); var withinDay = new List(); for (var i = 1; i < ordered.Count; i++) { var gap = (ordered[i].StartedAt - ordered[i - 1].StartedAt).TotalMinutes; intervals.Add(gap); if (LocalCalendar.Today(ordered[i].StartedAt, zone) == LocalCalendar.Today(ordered[i - 1].StartedAt, zone) && gap < 6 * 60) { withinDay.Add(gap); } } // Days. var firstDay = LocalCalendar.Today(start, zone); var lastDay = LocalCalendar.Today(end - TimeSpan.FromTicks(1), zone); var days = new List(); for (var day = firstDay; day <= lastDay; day = day.AddDays(1)) { var (dayStart, dayEnd) = LocalCalendar.DayBounds(day, zone); var daySessions = ordered.Where(x => x.StartedAt >= dayStart && x.StartedAt < dayEnd).ToList(); var active = daySessions.Sum(x => ((x.EndedAt ?? x.StartedAt) - x.StartedAt).TotalMinutes); var counts = Categories.ToDictionary(k => k, k => actions.Where(a => a.Category == k && a.At >= dayStart && a.At < dayEnd).Sum(a => a.Amount)); var worst = Categories.ToDictionary(k => k, k => worstHourByDay.GetValueOrDefault((day, k))); days.Add(new SimulatedDay( day, daySessions.Count, active, counts, worst, throttles.Count(t => t.At >= dayStart && t.At < dayEnd), overrunsByDay.GetValueOrDefault(day))); } var categories = Categories.Select(k => new SimulatedCategory( k, days.Count == 0 ? 0 : days.Average(d => (double)d.Actions[k]), days.Count == 0 ? 0 : days.Max(d => d.Actions[k]), worstHour.GetValueOrDefault(k), k == ActionKind.DataDownload ? null : profile.Budgets[k].PerHour, k == ActionKind.DataDownload ? profile.DataPerDayBytes : profile.Budgets[k].PerDay)).ToList(); var stats = world.Stats; return new SimulationReport { Profile = profile.Kind, Collection = world.Collection, Seed = options.Seed, Start = start, End = end, Zone = zone, Accounts = world.Accounts.Count, FollowedAccounts = world.Accounts.Count(a => a.Followed), Days = days, CategoryTotals = categories, SessionCount = ordered.Count, SessionsPerDayMean = days.Count == 0 ? 0 : days.Average(d => d.Sessions), SessionsPerDayMin = days.Count == 0 ? 0 : days.Min(d => d.Sessions), SessionsPerDayMax = days.Count == 0 ? 0 : days.Max(d => d.Sessions), MeanSessionLength = ordered.Count == 0 ? TimeSpan.Zero : TimeSpan.FromTicks((long)ordered.Average(x => ((x.EndedAt ?? x.StartedAt) - x.StartedAt).Ticks)), SessionStartCv = Cv(intervals), SessionStartCvWithinDay = Cv(withinDay), ShortestPause = shortest == TimeSpan.MaxValue ? TimeSpan.Zero : shortest, PauseFloorViolations = floorViolations, ActionsOutsideWindow = outside, FollowedPosts = DelayStats.From(stats.PostDiscoveries.Where(p => p.Followed).Select(p => p.Discovered - p.Published)), NotFollowedPosts = DelayStats.From(stats.PostDiscoveries.Where(p => !p.Followed).Select(p => p.Discovered - p.Published)), Stories = DelayStats.From(stats.StoryViews.Select(v => v.Seen - v.Published)), PublishedPosts = world.PublishedPosts(start, end), UndiscoveredPosts = world.UndiscoveredPosts(start, end), PublishedStories = world.PublishedStories(start, end), MissedStories = world.MissedStories(start, end), BackfillPosts = (options.World ?? new SimulatedWorldOptions()).HistoricalPosts, BackfillStored = stats.BackfillPostsStored, BackfillFinishedAt = stats.BackfillFinishedAt, Throttles = throttles, ProposalsOfMorePrudence = throttles.Count(t => throttles.Count(o => o.At <= t.At && o.At > t.At - CircuitBreaker.ProposalWindow) >= CircuitBreaker.ProposalThreshold), Overruns = overruns, OverrunsOfFullBudgets = overrunsFull, OverrunDetails = details, VirtualEvents = virtualEvents, World = stats, WorldAccounts = world.Accounts, RealDuration = realDuration, }; } /// Italian text report. public string ToText() { var b = new StringBuilder(); var firstDay = LocalCalendar.Today(Start, Zone); var lastDay = LocalCalendar.Today(End - TimeSpan.FromTicks(1), Zone); b.AppendLine($"Simulazione del motore: {ItalianText.Count(Days.Count, "giorno", "giorni")} con il profilo «{ItalianText.Profile(Profile)}» (seme {Seed.ToString(CultureInfo.InvariantCulture)})"); b.AppendLine(Collection == SimulatedCollection.Requests ? "Raccolta: per richieste" : "Raccolta: per pagine"); b.AppendLine($"Periodo: dal {ItalianText.Day(firstDay)} alle {ItalianText.Clock(Start, Zone)} al {ItalianText.Day(lastDay)}, fuso {Zone.Id}"); b.AppendLine($"Mondo finto: {ItalianText.Number(Accounts)} account ({ItalianText.Number(FollowedAccounts)} seguiti, {ItalianText.Number(Accounts - FollowedAccounts)} no), {ItalianText.Number(PublishedPosts)} post e {ItalianText.Number(PublishedStories)} storie pubblicati nel periodo, uno storico di {ItalianText.Number(BackfillPosts)} post"); b.AppendLine("I tetti del profilo sono stime prudenti, non limiti ufficiali di Instagram."); b.AppendLine(); b.AppendLine("Sessioni"); b.AppendLine($" al giorno: media {ItalianText.Decimal(SessionsPerDayMean, 1)}, minimo {ItalianText.Number(SessionsPerDayMin)}, massimo {ItalianText.Number(SessionsPerDayMax)} (in tutto {ItalianText.Number(SessionCount)})"); b.AppendLine($" durata media: {ItalianText.Duration(MeanSessionLength)}; pausa più breve: {ItalianText.Duration(ShortestPause)}"); b.AppendLine($" irregolarità degli inizi (coefficiente di variazione): {ItalianText.Decimal(SessionStartCv, 2)} in tutto, {ItalianText.Decimal(SessionStartCvWithinDay, 2)} dentro la giornata"); b.AppendLine(); b.AppendLine("Azioni per categoria: media al giorno, giorno peggiore, ora peggiore (tetto all'ora / al giorno)"); foreach (var c in CategoryTotals) { if (c.Kind == ActionKind.DataDownload) { b.AppendLine($" {ItalianText.CategoryLabel(c.Kind)}: {ItalianText.Bytes((long)c.MeanPerDay)} al giorno, giorno peggiore {ItalianText.Bytes(c.MaxPerDay)} (tetto {ItalianText.Bytes(c.DailyLimit)} al giorno)"); } else { b.AppendLine($" {ItalianText.CategoryLabel(c.Kind)}: {ItalianText.Decimal(c.MeanPerDay, 1)} al giorno, giorno peggiore {ItalianText.Number(c.MaxPerDay)}, ora peggiore {ItalianText.Number(c.WorstHour)} (tetto {ItalianText.Number(c.HourlyLimit ?? 0)} / {ItalianText.Number(c.DailyLimit)})"); } } b.AppendLine(); b.AppendLine("Dalla pubblicazione alla scoperta"); b.AppendLine($" post degli account seguiti: {Delay(FollowedPosts)}"); b.AppendLine($" post degli account non seguiti: {Delay(NotFollowedPosts)}"); b.AppendLine($" storie: {Delay(Stories)}"); b.AppendLine($" post non ancora scoperti alla fine: {ItalianText.Number(UndiscoveredPosts)} di {ItalianText.Number(PublishedPosts)}; storie scadute senza essere viste: {ItalianText.Number(MissedStories)} di {ItalianText.Number(PublishedStories)}"); b.AppendLine(); b.AppendLine($"Storico di {ItalianText.Number(BackfillPosts)} post"); b.AppendLine(BackfillFinishedAt is { } done ? $" completato in {ItalianText.Decimal(BackfillDays ?? 0, 1)} giorni ({ItalianText.DayWithWeekday(LocalCalendar.Today(done, Zone))} {ItalianText.At(done, Zone)})" : $" non completato: {ItalianText.Number(BackfillStored)} post scaricati in {ItalianText.Count(Days.Count, "giorno", "giorni")}"); b.AppendLine(); b.AppendLine("Rallentamenti simulati"); if (Throttles.Count == 0) { b.AppendLine(" nessuno"); } foreach (var t in Throttles) { var stop = t.StoppedUntil is { } until ? ItalianText.Duration(until - t.At) : "?"; b.AppendLine($" {ItalianText.DayWithWeekday(LocalCalendar.Today(t.At, Zone))} {ItalianText.At(t.At, Zone)}: fermo di {stop}"); } if (ProposalsOfMorePrudence > 0) { b.AppendLine($" proposte di passare a un profilo più prudente (tre rallentamenti in sette giorni): {ItalianText.Number(ProposalsOfMorePrudence)}"); } b.AppendLine(); b.AppendLine("Controlli"); b.AppendLine($" superamenti dei tetti: {ItalianText.Number(Overruns)} (dei tetti pieni del profilo: {ItalianText.Number(OverrunsOfFullBudgets)})"); foreach (var detail in OverrunDetails) { b.AppendLine(" " + detail); } b.AppendLine($" azioni fuori dalla fascia oraria: {ItalianText.Number(ActionsOutsideWindow)}"); b.AppendLine($" pause più brevi della pausa minima: {ItalianText.Number(PauseFloorViolations)}"); b.AppendLine(); b.AppendLine($"Simulati {ItalianText.Number(VirtualEvents)} eventi in {ItalianText.Decimal(RealDuration.TotalSeconds, 1)} secondi."); return b.ToString(); } /// One row per day, ';' separated, for Excel in Italian (write it as UTF-8 with BOM). public string ToCsv() { var b = new StringBuilder(); var headers = new List { "giorno", "sessioni", "minuti di attività" }; headers.AddRange(Categories.Where(k => k != ActionKind.DataDownload).Select(k => ItalianText.Category(k))); headers.Add("dati scaricati (MB)"); headers.AddRange(Categories.Where(k => k != ActionKind.DataDownload).Select(k => "ora peggiore, " + ItalianText.Category(k))); headers.Add("rallentamenti"); headers.Add("superamenti"); b.AppendLine(string.Join(';', headers.Select(Quote))); foreach (var day in Days) { var cells = new List { day.Day.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), day.Sessions.ToString(CultureInfo.InvariantCulture), ItalianText.Decimal(day.ActiveMinutes, 1, keepZeros: true).Replace(".", string.Empty, StringComparison.Ordinal), }; cells.AddRange(Categories.Where(k => k != ActionKind.DataDownload).Select(k => day.Actions[k].ToString(CultureInfo.InvariantCulture))); cells.Add(ItalianText.Decimal(day.Actions[ActionKind.DataDownload] / (1024.0 * 1024.0), 1, keepZeros: true).Replace(".", string.Empty, StringComparison.Ordinal)); cells.AddRange(Categories.Where(k => k != ActionKind.DataDownload).Select(k => day.WorstHour[k].ToString(CultureInfo.InvariantCulture))); cells.Add(day.Throttles.ToString(CultureInfo.InvariantCulture)); cells.Add(day.Overruns.ToString(CultureInfo.InvariantCulture)); b.AppendLine(string.Join(';', cells)); } return b.ToString(); } /// Writes as UTF-8 with BOM. public async Task WriteCsvAsync(string path, CancellationToken ct = default) { ArgumentException.ThrowIfNullOrWhiteSpace(path); await File.WriteAllTextAsync(path, ToCsv(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: true), ct).ConfigureAwait(false); } private static string Delay(DelayStats stats) => stats.Count == 0 ? "nessun caso" : $"media {ItalianText.Duration(stats.Mean)}, massimo {ItalianText.Duration(stats.Max)} ({ItalianText.Count(stats.Count, "caso", "casi")})"; private static string Quote(string text) => text.Contains(';', StringComparison.Ordinal) || text.Contains('"', StringComparison.Ordinal) ? "\"" + text.Replace("\"", "\"\"", StringComparison.Ordinal) + "\"" : text; private static double Cv(IReadOnlyList values) { if (values.Count < 2) { return 0; } var mean = values.Average(); if (mean <= 0) { return 0; } var variance = values.Sum(v => (v - mean) * (v - mean)) / (values.Count - 1); return Math.Sqrt(variance) / mean; } }