using System.Collections.Concurrent;
using System.Diagnostics;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
namespace Chronos.Data.Projects;
/// What a reconcile of a sequence's projects met ( ).
/// The sequence has no projects folder: nothing was read, and nothing taken for gone.
/// The projects the archive has a folder for, live or in the projects' trash, whether or not its record read.
/// The live projects the index could not be brought in line with: a record that cannot be read (damaged,
/// of a newer Chronos, a read failure), a head whose version this Chronos cannot read, or a row the index refused. Each stays
/// as the index has it, and nothing is written for it until it reads again.
/// The records of the live projects read, by id (their head as moved forward).
internal sealed record ProjectReconcile(bool Missing, IReadOnlySet Seen, IReadOnlySet HeldBack, IReadOnlyDictionary Live)
{
/// Every live project of the folder was read and indexed: a sequence without a Main may be given one.
public bool Complete => HeldBack.Count == 0;
}
///
/// Brings the index's projects of one sequence in line with their sidecars (owner review 2, R8), which are the truth: a
/// project or version the index lacks is added (an index rebuilt, a sequence restored from the trash), a head the index is
/// behind on moves forward, a name or deletion recorded since is taken, and a row the index marked deleted is listed again
/// when its live record says it is not. Only a record that says so, live or in the projects' trash, deletes a project:
/// never a folder that is not there (moved for a moment, a backup put back, a sync tool) nor a record that cannot be read,
/// whose project stays as the index has it ( ). The projects' trash is read before the
/// live folders, so a deletion is never undone by what is left of a live folder. A record missing from the folder of a live
/// project the index holds is written again from the index when the archive takes writes, once the folder has settled and
/// never while the trash holds a folder of that project. A record older than what the index knows (behind its head, or
/// written before it) never gives the index its name. A version file written past its
/// record's head by a push a crash cut short is adopted when it follows the head (its file is complete: versions are
/// written atomically, then the record); any other file past the head, or a second file of a number the record's chain
/// names another for, is moved aside to versions/.stray , never deleted. Runs under the sequence's lease, or without
/// writing to the archive at all when it is read-only.
///
internal sealed class ProjectIndexer(ChronosDatabase db)
{
private const string StrayFolder = ".stray";
/// Versions read at once when a project's sidecars are written again from the index.
private const int WriteBackPage = 500;
///
/// The live folders with no record seen by a reconcile, by project: of which sequence, what they held then, and when
/// ( ). A reconcile of the sequence that no longer meets the folder so forgets it.
///
private readonly ConcurrentDictionary _unsettled = new();
///
/// How long a folder of the projects' trash with no record (a deletion on its way, the record not there yet) holds back the
/// rebuild of a lost record, counted from the newest thing written in it: past that it is a leftover, not a deletion.
///
internal static readonly TimeSpan TrashRecordGrace = TimeSpan.FromDays(1);
///
/// How long a live folder with no record must look the same before its record is written again from the index (a sync
/// tool bringing the real one has it win meanwhile). Tests stand in a shorter one.
///
internal TimeSpan RecordSettle { get; set; } = TimeSpan.FromMinutes(2);
/// Reconciles the projects of in ; the caller holds the sequence's lease.
/// False for a read-only archive: the sidecars are read as they are and only the index is written
/// (a version past its record's head is indexed, not adopted into the record; nothing is moved aside).
public async Task ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken, bool writable = true)
{
var seen = new HashSet();
var heldBack = new HashSet();
var live = new Dictionary();
// The projects of this sequence whose record in the projects' trash was read, with the newest such folder.
var deletions = new Dictionary();
var unrecorded = new HashSet();
if (!Directory.Exists(folder.ProjectsDirectory)) return new ProjectReconcile(true, seen, heldBack, live);
foreach (var (directory, trashed) in ProjectFolders(folder))
{
cancellationToken.ThrowIfCancellationRequested();
// What is left of the live folder of a project whose deletion was read (the trash is read first): skipped, the
// versions it holds beyond the trash's moved there, beside the deleted record (a push made before the deletion
// reached this PC).
if (!trashed && Guid.TryParseExact(Path.GetFileName(directory), "N", out var left) && deletions.TryGetValue(left, out var deletion))
{
if (writable) KeepWithDeletion(directory, deletion, left);
continue;
}
ProjectRecord? record = null;
var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile);
try
{
record = ProjectSidecars.ReadRecord(recordPath);
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Its folder names it (, or - in the trash).
var name = Path.GetFileName(directory);
if (!Guid.TryParseExact(trashed && name.Length > 32 ? name[..32] : name, "N", out var named)) continue;
if (!trashed && !File.Exists(recordPath))
{
var row = await db.Projects.GetAsync(named, cancellationToken).ConfigureAwait(false);
if (row is null)
{
// Versions with no record, of a project no index row names: a creation a crash or a lost disk cut short before
// its record (versions are written first), or one a sync tool is still bringing. No project of the sequence is
// hidden in it, so it holds nothing back; it is left as it is, and indexed once its record is there.
Log.Info("projects", $"{directory} holds no project record; it is left as it is");
continue;
}
// A record lost (a sync tool, a disk) of a live project the index holds: written again from the index, once the
// folder has settled and no deletion of it is on its way (Rebuildable).
if (writable && row is { DeletedAt: null } && row.SequenceId == sequenceId)
{
unrecorded.Add(named);
if (Rebuildable(folder, directory, named, sequenceId)) record = await RecordFromIndexAsync(folder, row, recordPath, cancellationToken).ConfigureAwait(false);
}
}
if (record is null)
{
Log.Warning("projects", $"The project in {directory} cannot be read; it stays as the index has it", ex);
// Its row stays as it is, never taken for gone nor written again, and a live one is held back (it may be Main):
// also when the record that cannot be read is in the projects' trash, which may say a project the index has
// live was deleted (the trash is read first, so its live folder is not taken either).
if (seen.Add(named) && (!trashed || await LiveHereAsync(named, sequenceId, cancellationToken).ConfigureAwait(false))) heldBack.Add(named);
continue;
}
}
// A sequence folder copied by hand carries the projects of its source, which stay that sequence's. (A live folder
// with a record now is no longer unsettled: forgotten after the loop.)
if (record.SequenceId != sequenceId) continue;
if (trashed) deletions[record.Id] = directory;
if (!seen.Add(record.Id)) continue;
if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) };
if (record.Deleted is null)
{
record = RollForward(folder, record, directory, writable);
live[record.Id] = record;
}
try
{
if (!await IndexAsync(record, directory, trashed, writable, cancellationToken).ConfigureAwait(false)) heldBack.Add(record.Id);
}
catch (DatabaseException ex) when (ex.IsUniqueViolation || ex.IsForeignKeyViolation)
{
// One project the index refuses never stops the others, nor a scan of the archive: it is read again next time.
Log.Warning("projects", $"Project {record.Id} of sequence {sequenceId} could not be indexed", ex);
if (record.Deleted is null) heldBack.Add(record.Id);
}
}
// A folder with no record this sequence no longer has (it got one, or went): what was seen of it is forgotten.
foreach (var (id, unsettled) in _unsettled)
{
if (unsettled.Sequence == sequenceId && !unrecorded.Contains(id)) _unsettled.TryRemove(id, out _);
}
try
{
await SettleMainAsync(folder, sequenceId, live, heldBack, writable, cancellationToken).ConfigureAwait(false);
}
catch (DatabaseException ex) when (ex.IsUniqueViolation || ex.IsForeignKeyViolation)
{
Log.Warning("projects", $"The Main of sequence {sequenceId} could not be taken from its records; tried again next time", ex);
}
return new ProjectReconcile(false, seen, heldBack, live);
}
///
/// Stores and , Main passing from the first to the second, each
/// with its new name, in one transaction: the first goes through a name no project has, so no two live projects of the
/// sequence ever hold one name, nor both Main.
///
internal static async Task SwapMainAsync(ChronosDatabase db, SequenceProject demoted, SequenceProject promoted, CancellationToken cancellationToken)
{
await using var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false);
await tx.Projects.UpdateAsync(demoted with { IsMain = false, Name = "~" + demoted.Id.ToString("N") }, cancellationToken).ConfigureAwait(false);
await tx.Projects.UpdateAsync(promoted with { IsMain = true }, cancellationToken).ConfigureAwait(false);
await tx.Projects.UpdateAsync(demoted with { IsMain = false }, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
///
/// The index's Main taken from the records, which say which project is Main. When several records in step say so (an
/// index that met the sequence while Main's record was not there yet, a sync tool still bringing it, made a Main of its
/// own; another index passed Main on), the oldest of them that has a version (by the time its record says it was made,
/// then by id) is Main, the same in every index, and the others are ordinary projects, every version kept: in the index,
/// and in their records too when the archive takes writes, so each index and older Chronos reads them alike. Main and the
/// project it passes from take their records' names where free. Only records in step with the index count (a record
/// behind it, put back from an older backup, says what was): never while the index's Main is held back or its record is
/// not in step, and never more than one Main.
///
private async Task SettleMainAsync(SequencePaths folder, Guid sequenceId, Dictionary live, IReadOnlySet heldBack, bool writable,
CancellationToken cancellationToken)
{
var rows = await db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
ProjectRecord? InStep(SequenceProject row) =>
!heldBack.Contains(row.Id) && live.TryGetValue(row.Id, out var record) && record.Head?.Id == row.HeadVersionId ? record : null;
var main = rows.FirstOrDefault(p => p.IsMain);
var own = main is null ? null : InStep(main);
if (main is not null && own is null) return;
var claimants = rows.Where(p => p.HeadVersionId is not null && InStep(p) is { IsMain: true })
.OrderBy(p => live[p.Id].Created.UtcTicks).ThenBy(p => p.Id).ToList();
if (claimants.Count == 0) return;
var winner = claimants[0];
if (winner.Id != main?.Id)
{
var others = rows.Where(p => p.Id != winner.Id && p.Id != main?.Id).ToList();
var promoted = winner with { IsMain = true, Name = FreeName(others, live[winner.Id].Name, winner.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
Log.Warning("projects", $"The records of sequence {sequenceId} name project {winner.Id} as Main{(main is null ? string.Empty : $", not {main.Id}")}; the index takes it");
if (main is null)
{
await db.Projects.UpdateAsync(promoted, cancellationToken).ConfigureAwait(false);
}
else
{
var demoted = main with { IsMain = false, Name = FreeName([.. others, promoted], own!.Name, main.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
await SwapMainAsync(db, demoted, promoted, cancellationToken).ConfigureAwait(false);
}
}
if (!writable || claimants.Count < 2) return;
// The records that say Main and are not: ordinary projects, as the index names them.
foreach (var loser in claimants.Skip(1))
{
if (await db.Projects.GetAsync(loser.Id, cancellationToken).ConfigureAwait(false) is not { IsMain: false, DeletedAt: null } row) continue;
var record = live[loser.Id] with { IsMain = false, Name = row.Name, Mirrored = null, Updated = UpdatedAfter(live[loser.Id].Updated) };
try
{
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(loser.Id), record);
live[loser.Id] = record;
Log.Warning("projects", $"The record of project {loser.Id} of sequence {sequenceId} said it was Main beside project {winner.Id}; it says it is '{row.Name}'");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The record of project {loser.Id} still says it is Main; tried again next time", ex);
}
}
}
///
/// The record of a live project the index holds, written again from the index when its folder has none (a sync tool, a
/// disk lost it): the version files the folder lacks first (those there are left as they are), then the record naming
/// the index's head and, for Main, the version this index mirrored. The versions the folder holds past that head are
/// then adopted as after any push a crash cut short. Only once . Null when it cannot be written
/// now (tried again next time).
///
private async Task RecordFromIndexAsync(SequencePaths folder, SequenceProject row, string path, CancellationToken cancellationToken)
{
try
{
await WriteVersionsAsync(db, folder, row.Id, 0, cancellationToken).ConfigureAwait(false);
var mirrored = row.IsMain && row.MirroredVersionId is { } id && await db.Projects.GetVersionAsync(id, cancellationToken).ConfigureAwait(false) is { } version
? new ProjectHead(version.Id, version.Number)
: null;
var record = ProjectRecord.Of(row, mirrored);
ProjectSidecars.WriteRecord(path, record);
Log.Warning("projects", $"Project {row.Id} of sequence {row.SequenceId} had no record in its folder; it is written again from the index");
return record;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The record of project {row.Id} could not be written again from the index; tried again next time", ex);
return null;
}
}
///
/// Whether 's head is one of the versions the index holds below its head (or it names none while
/// the index has one): the record is behind. A record whose head the index does not hold at its number (another chain, a
/// version this Chronos cannot read) is not: it is never written over.
///
internal static async Task BehindAsync(ChronosDatabase db, SequenceProject project, ProjectRecord record, CancellationToken cancellationToken)
{
if (project.HeadVersionId is null) return false;
if (record.Head is not { } head) return true;
return head.Number < project.HeadNumber && await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is { } version
&& version.ProjectId == project.Id && version.Number == head.Number;
}
///
/// The time a record written now from the index says it was updated: now, or just after (the
/// time the record it replaces says) when a clock behind the one that wrote it would go back. A record's time never goes
/// back, so the newest write of a project (a rename on another PC) is always seen as newer than the one before.
///
internal static DateTimeOffset UpdatedAfter(DateTimeOffset previous)
{
var now = DateTimeOffset.UtcNow;
return now > previous ? now : previous.AddMilliseconds(1);
}
/// The version files of a project numbered above written from the index (immutable: one there already is left).
internal static async Task WriteVersionsAsync(ChronosDatabase db, SequencePaths folder, Guid projectId, int after, CancellationToken cancellationToken)
{
while (true)
{
var page = await db.Projects.ListVersionsAfterAsync(projectId, after, WriteBackPage, cancellationToken).ConfigureAwait(false);
foreach (var version in page) ProjectSidecars.WriteVersion(folder.VersionPath(projectId, version.Number, version.Id), version);
if (page.Count < WriteBackPage) return;
after = page[^1].Number;
}
}
///
/// , or " (2)", "(3)"… (within ):
/// the first no project of but has.
///
internal static string FreeName(IReadOnlyList live, string name, Guid? projectId)
{
var taken = live.Where(p => p.Id != projectId).Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
var candidate = name;
for (var n = 2; taken.Contains(candidate); n++)
{
var suffix = $" ({n.ToString(CultureInfo.InvariantCulture)})";
candidate = (name.Length + suffix.Length > ProjectService.MaxNameLength ? name[..(ProjectService.MaxNameLength - suffix.Length)].TrimEnd() : name) + suffix;
}
return candidate;
}
///
/// The project folders of a sequence: those in the projects' trash (<id>-<time> ), then the live ones.
/// The trash comes first so a deletion wins over what is left of the project's live folder (a sync tool moving it into
/// the trash file by file, another PC's delete arriving): a deletion is never undone by a reconcile.
///
private static IEnumerable<(string Directory, bool Trashed)> ProjectFolders(SequencePaths folder)
{
if (Directory.Exists(folder.ProjectsTrashDirectory))
{
foreach (var directory in Directory.EnumerateDirectories(folder.ProjectsTrashDirectory).Order(StringComparer.Ordinal))
{
if (File.Exists(Path.Combine(directory, ArchiveLayout.ProjectFile))) yield return (directory, true);
}
}
foreach (var directory in Directory.EnumerateDirectories(folder.ProjectsDirectory).Order(StringComparer.Ordinal))
{
if (Guid.TryParseExact(Path.GetFileName(directory), "N", out _)) yield return (directory, false);
}
}
/// Whether the index holds project live, of this sequence.
private async Task LiveHereAsync(Guid projectId, Guid sequenceId, CancellationToken cancellationToken) =>
await db.Projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) is { DeletedAt: null } row && row.SequenceId == sequenceId;
///
/// Whether the live folder of project , which has no record, may
/// have its record written again from the index now. Never while the projects' trash holds a folder of it that may be a
/// deletion on its way ( ), and only once the folder has looked the same, file by file, on
/// two reconciles apart: a sync tool still bringing the real record (with a rename, a newer
/// head) has it win. Until then the project is held back.
///
private bool Rebuildable(SequencePaths folder, string directory, Guid projectId, Guid sequenceId)
{
var prefix = projectId.ToString("N") + "-";
try
{
if (Directory.Exists(folder.ProjectsTrashDirectory)
&& Directory.EnumerateDirectories(folder.ProjectsTrashDirectory)
.Any(d => Path.GetFileName(d).StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && DeletionOnItsWay(d, projectId, sequenceId, DateTime.UtcNow)))
{
Log.Info("projects", $"Project {projectId} has a folder in the projects' trash; its record is not written again");
return false;
}
var look = Look(directory);
var now = Stopwatch.GetTimestamp();
if (_unsettled.TryGetValue(projectId, out var earlier) && earlier.Look == look)
{
if (Stopwatch.GetElapsedTime(earlier.Since, now) < RecordSettle) return false;
_unsettled.TryRemove(projectId, out _);
return true;
}
_unsettled[projectId] = (sequenceId, look, now);
Log.Info("projects", $"Project {projectId} has no record in its folder; it is written again from the index once the folder settles");
return false;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The folder of project {projectId} cannot be read now", ex);
return false;
}
}
///
/// Whether the folder of the projects' trash, named for , may be
/// a deletion of this sequence's project on its way. Its record names the sequence (one of another, a sequence folder
/// copied by hand, is not), or cannot be read (nothing is guessed). Without a record (versions name no sequence) it is
/// one until has passed since anything was last written in it, unless a version in it is
/// another project's: a leftover then, which no longer holds a live project back.
///
internal static bool DeletionOnItsWay(string directory, Guid projectId, Guid sequenceId, DateTime nowUtc)
{
var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile);
if (File.Exists(recordPath))
{
try
{
return ProjectSidecars.ReadRecord(recordPath).SequenceId == sequenceId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
return true;
}
}
var info = new DirectoryInfo(directory);
var newest = info.LastWriteTimeUtc;
foreach (var entry in info.EnumerateFileSystemInfos("*", SearchOption.AllDirectories))
{
if (entry.LastWriteTimeUtc > newest) newest = entry.LastWriteTimeUtc;
}
if (nowUtc - newest >= TrashRecordGrace) return false;
foreach (var (_, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder)))
{
try
{
if (ProjectSidecars.ReadVersion(path).ProjectId != projectId) return false;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Not read: proves nothing either way.
}
}
return true;
}
///
/// The version files the live folder of a deleted project holds that its folder in the
/// projects' trash ( ) lacks, moved there beside the deleted record: a version pushed into the
/// live folder before the deletion reached this PC is kept with the project, never deleted nor left where nothing reads
/// it. A file the trash has under the same name is left as it is.
///
private static void KeepWithDeletion(string directory, string deletion, Guid projectId)
{
var target = Path.Combine(deletion, ArchiveLayout.ProjectVersionsFolder);
foreach (var (number, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder)))
{
var to = Path.Combine(target, Path.GetFileName(path));
try
{
if (File.Exists(to) || ProjectSidecars.ReadVersion(path).ProjectId != projectId) continue;
Directory.CreateDirectory(target);
File.Move(path, to);
Log.Warning("projects", $"Version {number} of deleted project {projectId} was in its live folder; it is kept beside its deleted record in {deletion}");
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {path} of deleted project {projectId} could not be moved beside its deleted record; tried again next time", ex);
}
}
}
/// What a project folder holds now, file by file (path, size, time written), as one hash.
private static string Look(string directory)
{
var text = new StringBuilder();
foreach (var file in new DirectoryInfo(directory).EnumerateFiles("*", SearchOption.AllDirectories).OrderBy(f => f.FullName, StringComparer.Ordinal))
{
text.Append(Path.GetRelativePath(directory, file.FullName)).Append('|').Append(file.Length.ToString(CultureInfo.InvariantCulture)).Append('|')
.Append(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)).Append('\n');
}
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString())));
}
///
/// The record with the version files that follow its head adopted (a push that wrote its version but not the record),
/// and the files past the head that do not follow it moved aside; the record is written again when its head moved. A
/// file past the head that cannot be read proves nothing: then none is moved aside.
///
private static ProjectRecord RollForward(SequencePaths folder, ProjectRecord record, string directory, bool writable)
{
var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
var files = ProjectSidecars.ListVersions(versions);
var head = record.Head;
var moved = false;
var unread = false;
while (true)
{
var next = (head?.Number ?? 0) + 1;
ProjectVersion? follower = null;
foreach (var candidate in files.Where(f => f.Number == next))
{
try
{
var version = ProjectSidecars.ReadVersion(candidate.Path);
if (version.ProjectId == record.Id && version.Id == candidate.Id && version.ParentId == head?.Id)
{
follower = version;
break;
}
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {candidate.Path} cannot be read", ex);
unread = true;
}
}
if (follower is null) break;
head = new ProjectHead(follower.Id, follower.Number);
moved = true;
}
if (writable && !unread)
{
foreach (var stray in files.Where(f => f.Number > (head?.Number ?? 0)))
SetAside(versions, stray.Path, $"does not follow the head of project {record.Id}");
}
if (!moved) return record;
var rolled = record with { Head = head, Updated = UpdatedAfter(record.Updated) };
if (!writable) return rolled;
try
{
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(record.Id), rolled);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// The index takes the versions all the same; the record is written with them before anything else is (ProjectService).
Log.Warning("projects", $"The record of project {record.Id} could not be moved forward to version {head!.Number}", ex);
}
return rolled;
}
/// A version file moved aside into versions/.stray (kept, never deleted).
private static void SetAside(string versions, string path, string why)
{
try
{
var aside = Path.Combine(versions, StrayFolder);
Directory.CreateDirectory(aside);
File.Move(path, Path.Combine(aside, Path.GetFileName(path) + "." + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff", CultureInfo.InvariantCulture)));
Log.Warning("projects", $"Version file {path} {why}; moved aside");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {path} could not be moved aside", ex);
}
}
///
/// Writes what the index lacks of one project's record and versions. False when its head could not be reached: the
/// record names a version this Chronos cannot read (damaged, of a newer Chronos), and the index's head stays on the
/// newest one it holds below it.
///
private async Task IndexAsync(ProjectRecord record, string directory, bool trashed, bool writable, CancellationToken cancellationToken)
{
var row = await db.Projects.GetAsync(record.Id, cancellationToken).ConfigureAwait(false);
if (row is null)
{
var project = record.ToProject();
if (project.DeletedAt is null)
{
var live = await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
if (project.IsMain && live.Any(p => p.IsMain))
{
Log.Warning("projects", $"Project {record.Id} says it is Main, but sequence {record.SequenceId} has one; indexed as another project");
project = project with { IsMain = false };
}
project = project with { Name = FreeName(live, project.Name, project.Id) };
}
// The head follows its versions below, so a version the index cannot take never leaves a head without its row.
row = project with { HeadVersionId = null, HeadNumber = 0 };
await db.Projects.InsertAsync(row, cancellationToken).ConfigureAwait(false);
}
// A record behind the index (a projects folder put back from an older backup: its head is a version the index holds
// below its own, or it was written before what the index knows of the project, as a rename made since) says what was:
// its name is not taken, and the heal writes the index's back (ProjectService). A record on another chain is not behind.
var stale = await BehindAsync(db, row, record, cancellationToken).ConfigureAwait(false) || record.Updated < row.UpdatedAt;
await IndexVersionsAsync(record, directory, writable, cancellationToken).ConfigureAwait(false);
var (headId, headNumber) = (row.HeadVersionId, row.HeadNumber);
var reached = true;
if (record.Head is { } head && (head.Number > headNumber || (head.Number == headNumber && head.Id != headId)))
{
if (await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.ProjectId == record.Id
&& indexed.Number == head.Number)
{
(headId, headNumber) = (head.Id, head.Number);
}
else
{
reached = false;
Log.Warning("projects", $"The newest version of project {record.Id} ({head.Number}) cannot be read; the project is shown as far as it can be, and not changed");
var below = await db.Projects.ListVersionsAsync(record.Id, head.Number, 1, cancellationToken).ConfigureAwait(false);
if (below.Count > 0 && below[0].Number > headNumber) (headId, headNumber) = (below[0].Id, below[0].Number);
}
}
var updated = row with { HeadVersionId = headId, HeadNumber = headNumber };
if (record.Deleted is not null || trashed)
{
updated = updated with { DeletedAt = row.DeletedAt ?? record.Deleted };
}
else if (row.DeletedAt is not null || (row.Name != record.Name && !stale))
{
var others = (await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
.Where(p => p.Id != record.Id).ToList();
if (row.DeletedAt is not null)
{
// Its live record says it is not deleted: listed again, Main again unless the sequence has another.
Log.Warning("projects", $"Project {record.Id} of sequence {record.SequenceId} is live in its sidecars; it is listed again");
updated = updated with { DeletedAt = null, IsMain = row.IsMain && !others.Any(p => p.IsMain) };
}
// The record's name (unless it is behind), else (another project has it) the name the index has, else one free.
updated = updated with
{
Name = !stale && FreeName(others, record.Name, record.Id) == record.Name ? record.Name
: FreeName(others, row.Name, record.Id) == row.Name ? row.Name
: FreeName(others, stale ? row.Name : record.Name, record.Id),
};
// What the index knows of the project is now as new as the record, so an older one put back later is seen as such.
if (!stale && record.Updated > updated.UpdatedAt) updated = updated with { UpdatedAt = record.Updated };
}
if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false);
return reached || record.Deleted is not null || trashed;
}
///
/// The version files up to the record's head the index lacks, taken in. Of several files of one number, the one the
/// record's chain names is taken and the others are moved aside; when the chain cannot tell (a version on it cannot be
/// read), none is taken.
///
private async Task IndexVersionsAsync(ProjectRecord record, string directory, bool writable, CancellationToken cancellationToken)
{
var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
var files = ProjectSidecars.ListVersions(versions);
var limit = record.Head?.Number ?? 0;
var indexed = new Dictionary(await db.Projects.ListVersionIdsAsync(record.Id, cancellationToken).ConfigureAwait(false));
Dictionary? chain = null;
foreach (var number in files.Where(f => f.Number <= limit && !indexed.ContainsKey(f.Number)).GroupBy(f => f.Number))
{
var candidates = number.ToList();
if (candidates.Count > 1)
{
chain ??= Chain(record, files);
var named = candidates.Where(c => chain.TryGetValue(c.Number, out var id) && id == c.Id).ToList();
if (named.Count == 0)
{
Log.Warning("projects", $"Project {record.Id} has {candidates.Count} version files numbered {number.Key} and its record's chain names none; none is indexed");
continue;
}
if (writable)
{
foreach (var other in candidates.Where(c => c.Id != named[0].Id))
SetAside(versions, other.Path, $"has the number of another version of project {record.Id}");
}
candidates = named;
}
foreach (var file in candidates)
{
ProjectVersion version;
try
{
version = ProjectSidecars.ReadVersion(file.Path);
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {file.Path} cannot be read; it is not indexed", ex);
continue;
}
if (version.ProjectId != record.Id || version.Number != file.Number || version.Id != file.Id) continue;
if (await db.Projects.GetVersionAsync(version.Id, cancellationToken).ConfigureAwait(false) is not null)
{
Log.Warning("projects", $"Version file {file.Path} holds a version the index has under another number; it is not indexed");
continue;
}
await db.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
indexed[version.Number] = version.Id;
}
}
}
/// The version ids by number along the record's chain, from its head back through each parent, as far as their files read.
private static Dictionary Chain(ProjectRecord record, IReadOnlyList<(int Number, Guid Id, string Path)> files)
{
var chain = new Dictionary();
var byId = files.GroupBy(f => f.Id).ToDictionary(g => g.Key, g => g.First());
for (var id = record.Head?.Id; id is { } current && byId.TryGetValue(current, out var file) && !chain.ContainsKey(file.Number);)
{
chain[file.Number] = file.Id;
try
{
id = ProjectSidecars.ReadVersion(file.Path).ParentId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {file.Path} cannot be read", ex);
break;
}
}
return chain;
}
}