using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; using System.Security.Cryptography; using System.Text; using Chronos.Core.Archiving; using Chronos.Core.Diagnostics; using Chronos.Core.Model; namespace Chronos.Data.Projects; /// What a reconcile of a sequence's projects met (). /// The sequence has no projects folder: nothing was read, and nothing taken for gone. /// The projects the archive has a folder for, live or in the projects' trash, whether or not its record read. /// The live projects the index could not be brought in line with: a record that cannot be read (damaged, /// of a newer Chronos, a read failure), a head whose version this Chronos cannot read, or a row the index refused. Each stays /// as the index has it, and nothing is written for it until it reads again. /// The records of the live projects read, by id (their head as moved forward). internal sealed record ProjectReconcile(bool Missing, IReadOnlySet Seen, IReadOnlySet HeldBack, IReadOnlyDictionary Live) { /// Every live project of the folder was read and indexed: a sequence without a Main may be given one. public bool Complete => HeldBack.Count == 0; } /// /// Brings the index's projects of one sequence in line with their sidecars (owner review 2, R8), which are the truth: a /// project or version the index lacks is added (an index rebuilt, a sequence restored from the trash), a head the index is /// behind on moves forward, a name or deletion recorded since is taken, and a row the index marked deleted is listed again /// when its live record says it is not. Only a record that says so, live or in the projects' trash, deletes a project: /// never a folder that is not there (moved for a moment, a backup put back, a sync tool) nor a record that cannot be read, /// whose project stays as the index has it (). The projects' trash is read before the /// live folders, so a deletion is never undone by what is left of a live folder. A record missing from the folder of a live /// project the index holds is written again from the index when the archive takes writes, once the folder has settled and /// never while the trash holds a folder of that project. A record older than what the index knows (behind its head, or /// written before it) never gives the index its name. A version file written past its /// record's head by a push a crash cut short is adopted when it follows the head (its file is complete: versions are /// written atomically, then the record); any other file past the head, or a second file of a number the record's chain /// names another for, is moved aside to versions/.stray, never deleted. Runs under the sequence's lease, or without /// writing to the archive at all when it is read-only. /// internal sealed class ProjectIndexer(ChronosDatabase db) { private const string StrayFolder = ".stray"; /// Versions read at once when a project's sidecars are written again from the index. private const int WriteBackPage = 500; /// /// The live folders with no record seen by a reconcile, by project: of which sequence, what they held then, and when /// (). A reconcile of the sequence that no longer meets the folder so forgets it. /// private readonly ConcurrentDictionary _unsettled = new(); /// /// How long a folder of the projects' trash with no record (a deletion on its way, the record not there yet) holds back the /// rebuild of a lost record, counted from the newest thing written in it: past that it is a leftover, not a deletion. /// internal static readonly TimeSpan TrashRecordGrace = TimeSpan.FromDays(1); /// /// How long a live folder with no record must look the same before its record is written again from the index (a sync /// tool bringing the real one has it win meanwhile). Tests stand in a shorter one. /// internal TimeSpan RecordSettle { get; set; } = TimeSpan.FromMinutes(2); /// Reconciles the projects of in ; the caller holds the sequence's lease. /// False for a read-only archive: the sidecars are read as they are and only the index is written /// (a version past its record's head is indexed, not adopted into the record; nothing is moved aside). public async Task ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken, bool writable = true) { var seen = new HashSet(); var heldBack = new HashSet(); var live = new Dictionary(); // The projects of this sequence whose record in the projects' trash was read, with the newest such folder. var deletions = new Dictionary(); var unrecorded = new HashSet(); if (!Directory.Exists(folder.ProjectsDirectory)) return new ProjectReconcile(true, seen, heldBack, live); foreach (var (directory, trashed) in ProjectFolders(folder)) { cancellationToken.ThrowIfCancellationRequested(); // What is left of the live folder of a project whose deletion was read (the trash is read first): skipped, the // versions it holds beyond the trash's moved there, beside the deleted record (a push made before the deletion // reached this PC). if (!trashed && Guid.TryParseExact(Path.GetFileName(directory), "N", out var left) && deletions.TryGetValue(left, out var deletion)) { if (writable) KeepWithDeletion(directory, deletion, left); continue; } ProjectRecord? record = null; var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile); try { record = ProjectSidecars.ReadRecord(recordPath); } catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException) { // Its folder names it (, or -