Files
Chronos/src/Chronos.Data/Projects/ProjectIndexer.cs
T
Alby96andClaude Opus 5.5 7692ab402e fix(workspace): a cancelled copy clears only media no other copy writes; a deletion keeps late versions
Owner review 2, phase 5a, fix round 5.

- A cancelled workspace copy clears its partial media only under the project's
  lease, after checking again that the job is still cancelled, that no other
  copy of the same root and project is queued, running or paused, and that no
  copy in this process is writing. The engine now waits for a paused job's run
  to end before calling the cancel handler, so the clean-up is never skipped.
- The clean-up never recreates a project folder that was removed meanwhile
  (the lease no longer creates the folder). The next list removes a project
  folder that holds only lease files nobody holds, older than the lease.
- A Main this Chronos cannot read (newer or damaged) refuses the open at once
  only when nothing else can be offered. Otherwise Open in workspace lists it
  disabled, with the reason as its tooltip. The refusal's action tooltip is
  now Remove copy.
- The removal plan counts a conflict the same way the sync status does: a
  conflict that a settling replay took past its commit point is not work
  left.
- A healed record is stamped after both the record's time and the index's, so
  the next rename made elsewhere is taken.
- A trash folder holds back the rebuild of a lost record only when it is this
  sequence's deletion: its record names this sequence or cannot be read, or it
  has no record, is within a one-day grace and holds no version of another
  project. What was seen of folders that are gone or have a record again is
  forgotten.
- When a deletion wins, version files pushed into the live folder before the
  deletion arrived are moved beside the deleted record in the trash, never
  deleted.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 15:10:44 +01:00

684 lines
40 KiB
C#

using System.Collections.Concurrent;
using System.Diagnostics;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
namespace Chronos.Data.Projects;
/// <summary>What a reconcile of a sequence's projects met (<see cref="ProjectIndexer.ReconcileAsync"/>).</summary>
/// <param name="Missing">The sequence has no projects folder: nothing was read, and nothing taken for gone.</param>
/// <param name="Seen">The projects the archive has a folder for, live or in the projects' trash, whether or not its record read.</param>
/// <param name="HeldBack">The live projects the index could not be brought in line with: a record that cannot be read (damaged,
/// of a newer Chronos, a read failure), a head whose version this Chronos cannot read, or a row the index refused. Each stays
/// as the index has it, and nothing is written for it until it reads again.</param>
/// <param name="Live">The records of the live projects read, by id (their head as moved forward).</param>
internal sealed record ProjectReconcile(bool Missing, IReadOnlySet<Guid> Seen, IReadOnlySet<Guid> HeldBack, IReadOnlyDictionary<Guid, ProjectRecord> Live)
{
/// <summary>Every live project of the folder was read and indexed: a sequence without a Main may be given one.</summary>
public bool Complete => HeldBack.Count == 0;
}
/// <summary>
/// Brings the index's projects of one sequence in line with their sidecars (owner review 2, R8), which are the truth: a
/// project or version the index lacks is added (an index rebuilt, a sequence restored from the trash), a head the index is
/// behind on moves forward, a name or deletion recorded since is taken, and a row the index marked deleted is listed again
/// when its live record says it is not. Only a record that says so, live or in the projects' trash, deletes a project:
/// never a folder that is not there (moved for a moment, a backup put back, a sync tool) nor a record that cannot be read,
/// whose project stays as the index has it (<see cref="ProjectReconcile.HeldBack"/>). The projects' trash is read before the
/// live folders, so a deletion is never undone by what is left of a live folder. A record missing from the folder of a live
/// project the index holds is written again from the index when the archive takes writes, once the folder has settled and
/// never while the trash holds a folder of that project. A record older than what the index knows (behind its head, or
/// written before it) never gives the index its name. A version file written past its
/// record's head by a push a crash cut short is adopted when it follows the head (its file is complete: versions are
/// written atomically, then the record); any other file past the head, or a second file of a number the record's chain
/// names another for, is moved aside to <c>versions/.stray</c>, never deleted. Runs under the sequence's lease, or without
/// writing to the archive at all when it is read-only.
/// </summary>
internal sealed class ProjectIndexer(ChronosDatabase db)
{
private const string StrayFolder = ".stray";
/// <summary>Versions read at once when a project's sidecars are written again from the index.</summary>
private const int WriteBackPage = 500;
/// <summary>
/// The live folders with no record seen by a reconcile, by project: of which sequence, what they held then, and when
/// (<see cref="Rebuildable"/>). A reconcile of the sequence that no longer meets the folder so forgets it.
/// </summary>
private readonly ConcurrentDictionary<Guid, (Guid Sequence, string Look, long Since)> _unsettled = new();
/// <summary>
/// How long a folder of the projects' trash with no record (a deletion on its way, the record not there yet) holds back the
/// rebuild of a lost record, counted from the newest thing written in it: past that it is a leftover, not a deletion.
/// </summary>
internal static readonly TimeSpan TrashRecordGrace = TimeSpan.FromDays(1);
/// <summary>
/// How long a live folder with no record must look the same before its record is written again from the index (a sync
/// tool bringing the real one has it win meanwhile). Tests stand in a shorter one.
/// </summary>
internal TimeSpan RecordSettle { get; set; } = TimeSpan.FromMinutes(2);
/// <summary>Reconciles the projects of <paramref name="sequenceId"/> in <paramref name="folder"/>; the caller holds the sequence's lease.</summary>
/// <param name="writable">False for a read-only archive: the sidecars are read as they are and only the index is written
/// (a version past its record's head is indexed, not adopted into the record; nothing is moved aside).</param>
public async Task<ProjectReconcile> ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken, bool writable = true)
{
var seen = new HashSet<Guid>();
var heldBack = new HashSet<Guid>();
var live = new Dictionary<Guid, ProjectRecord>();
// The projects of this sequence whose record in the projects' trash was read, with the newest such folder.
var deletions = new Dictionary<Guid, string>();
var unrecorded = new HashSet<Guid>();
if (!Directory.Exists(folder.ProjectsDirectory)) return new ProjectReconcile(true, seen, heldBack, live);
foreach (var (directory, trashed) in ProjectFolders(folder))
{
cancellationToken.ThrowIfCancellationRequested();
// What is left of the live folder of a project whose deletion was read (the trash is read first): skipped, the
// versions it holds beyond the trash's moved there, beside the deleted record (a push made before the deletion
// reached this PC).
if (!trashed && Guid.TryParseExact(Path.GetFileName(directory), "N", out var left) && deletions.TryGetValue(left, out var deletion))
{
if (writable) KeepWithDeletion(directory, deletion, left);
continue;
}
ProjectRecord? record = null;
var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile);
try
{
record = ProjectSidecars.ReadRecord(recordPath);
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Its folder names it (<id>, or <id>-<time> in the trash).
var name = Path.GetFileName(directory);
if (!Guid.TryParseExact(trashed && name.Length > 32 ? name[..32] : name, "N", out var named)) continue;
if (!trashed && !File.Exists(recordPath))
{
var row = await db.Projects.GetAsync(named, cancellationToken).ConfigureAwait(false);
if (row is null)
{
// Versions with no record, of a project no index row names: a creation a crash or a lost disk cut short before
// its record (versions are written first), or one a sync tool is still bringing. No project of the sequence is
// hidden in it, so it holds nothing back; it is left as it is, and indexed once its record is there.
Log.Info("projects", $"{directory} holds no project record; it is left as it is");
continue;
}
// A record lost (a sync tool, a disk) of a live project the index holds: written again from the index, once the
// folder has settled and no deletion of it is on its way (Rebuildable).
if (writable && row is { DeletedAt: null } && row.SequenceId == sequenceId)
{
unrecorded.Add(named);
if (Rebuildable(folder, directory, named, sequenceId)) record = await RecordFromIndexAsync(folder, row, recordPath, cancellationToken).ConfigureAwait(false);
}
}
if (record is null)
{
Log.Warning("projects", $"The project in {directory} cannot be read; it stays as the index has it", ex);
// Its row stays as it is, never taken for gone nor written again, and a live one is held back (it may be Main):
// also when the record that cannot be read is in the projects' trash, which may say a project the index has
// live was deleted (the trash is read first, so its live folder is not taken either).
if (seen.Add(named) && (!trashed || await LiveHereAsync(named, sequenceId, cancellationToken).ConfigureAwait(false))) heldBack.Add(named);
continue;
}
}
// A sequence folder copied by hand carries the projects of its source, which stay that sequence's. (A live folder
// with a record now is no longer unsettled: forgotten after the loop.)
if (record.SequenceId != sequenceId) continue;
if (trashed) deletions[record.Id] = directory;
if (!seen.Add(record.Id)) continue;
if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) };
if (record.Deleted is null)
{
record = RollForward(folder, record, directory, writable);
live[record.Id] = record;
}
try
{
if (!await IndexAsync(record, directory, trashed, writable, cancellationToken).ConfigureAwait(false)) heldBack.Add(record.Id);
}
catch (DatabaseException ex) when (ex.IsUniqueViolation || ex.IsForeignKeyViolation)
{
// One project the index refuses never stops the others, nor a scan of the archive: it is read again next time.
Log.Warning("projects", $"Project {record.Id} of sequence {sequenceId} could not be indexed", ex);
if (record.Deleted is null) heldBack.Add(record.Id);
}
}
// A folder with no record this sequence no longer has (it got one, or went): what was seen of it is forgotten.
foreach (var (id, unsettled) in _unsettled)
{
if (unsettled.Sequence == sequenceId && !unrecorded.Contains(id)) _unsettled.TryRemove(id, out _);
}
try
{
await SettleMainAsync(folder, sequenceId, live, heldBack, writable, cancellationToken).ConfigureAwait(false);
}
catch (DatabaseException ex) when (ex.IsUniqueViolation || ex.IsForeignKeyViolation)
{
Log.Warning("projects", $"The Main of sequence {sequenceId} could not be taken from its records; tried again next time", ex);
}
return new ProjectReconcile(false, seen, heldBack, live);
}
/// <summary>
/// Stores <paramref name="demoted"/> and <paramref name="promoted"/>, Main passing from the first to the second, each
/// with its new name, in one transaction: the first goes through a name no project has, so no two live projects of the
/// sequence ever hold one name, nor both Main.
/// </summary>
internal static async Task SwapMainAsync(ChronosDatabase db, SequenceProject demoted, SequenceProject promoted, CancellationToken cancellationToken)
{
await using var tx = await db.BeginAsync(cancellationToken).ConfigureAwait(false);
await tx.Projects.UpdateAsync(demoted with { IsMain = false, Name = "~" + demoted.Id.ToString("N") }, cancellationToken).ConfigureAwait(false);
await tx.Projects.UpdateAsync(promoted with { IsMain = true }, cancellationToken).ConfigureAwait(false);
await tx.Projects.UpdateAsync(demoted with { IsMain = false }, cancellationToken).ConfigureAwait(false);
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// The index's Main taken from the records, which say which project is Main. When several records in step say so (an
/// index that met the sequence while Main's record was not there yet, a sync tool still bringing it, made a Main of its
/// own; another index passed Main on), the oldest of them that has a version (by the time its record says it was made,
/// then by id) is Main, the same in every index, and the others are ordinary projects, every version kept: in the index,
/// and in their records too when the archive takes writes, so each index and older Chronos reads them alike. Main and the
/// project it passes from take their records' names where free. Only records in step with the index count (a record
/// behind it, put back from an older backup, says what was): never while the index's Main is held back or its record is
/// not in step, and never more than one Main.
/// </summary>
private async Task SettleMainAsync(SequencePaths folder, Guid sequenceId, Dictionary<Guid, ProjectRecord> live, IReadOnlySet<Guid> heldBack, bool writable,
CancellationToken cancellationToken)
{
var rows = await db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
ProjectRecord? InStep(SequenceProject row) =>
!heldBack.Contains(row.Id) && live.TryGetValue(row.Id, out var record) && record.Head?.Id == row.HeadVersionId ? record : null;
var main = rows.FirstOrDefault(p => p.IsMain);
var own = main is null ? null : InStep(main);
if (main is not null && own is null) return;
var claimants = rows.Where(p => p.HeadVersionId is not null && InStep(p) is { IsMain: true })
.OrderBy(p => live[p.Id].Created.UtcTicks).ThenBy(p => p.Id).ToList();
if (claimants.Count == 0) return;
var winner = claimants[0];
if (winner.Id != main?.Id)
{
var others = rows.Where(p => p.Id != winner.Id && p.Id != main?.Id).ToList();
var promoted = winner with { IsMain = true, Name = FreeName(others, live[winner.Id].Name, winner.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
Log.Warning("projects", $"The records of sequence {sequenceId} name project {winner.Id} as Main{(main is null ? string.Empty : $", not {main.Id}")}; the index takes it");
if (main is null)
{
await db.Projects.UpdateAsync(promoted, cancellationToken).ConfigureAwait(false);
}
else
{
var demoted = main with { IsMain = false, Name = FreeName([.. others, promoted], own!.Name, main.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
await SwapMainAsync(db, demoted, promoted, cancellationToken).ConfigureAwait(false);
}
}
if (!writable || claimants.Count < 2) return;
// The records that say Main and are not: ordinary projects, as the index names them.
foreach (var loser in claimants.Skip(1))
{
if (await db.Projects.GetAsync(loser.Id, cancellationToken).ConfigureAwait(false) is not { IsMain: false, DeletedAt: null } row) continue;
var record = live[loser.Id] with { IsMain = false, Name = row.Name, Mirrored = null, Updated = UpdatedAfter(live[loser.Id].Updated) };
try
{
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(loser.Id), record);
live[loser.Id] = record;
Log.Warning("projects", $"The record of project {loser.Id} of sequence {sequenceId} said it was Main beside project {winner.Id}; it says it is '{row.Name}'");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The record of project {loser.Id} still says it is Main; tried again next time", ex);
}
}
}
/// <summary>
/// The record of a live project the index holds, written again from the index when its folder has none (a sync tool, a
/// disk lost it): the version files the folder lacks first (those there are left as they are), then the record naming
/// the index's head and, for Main, the version this index mirrored. The versions the folder holds past that head are
/// then adopted as after any push a crash cut short. Only once <see cref="Rebuildable"/>. Null when it cannot be written
/// now (tried again next time).
/// </summary>
private async Task<ProjectRecord?> RecordFromIndexAsync(SequencePaths folder, SequenceProject row, string path, CancellationToken cancellationToken)
{
try
{
await WriteVersionsAsync(db, folder, row.Id, 0, cancellationToken).ConfigureAwait(false);
var mirrored = row.IsMain && row.MirroredVersionId is { } id && await db.Projects.GetVersionAsync(id, cancellationToken).ConfigureAwait(false) is { } version
? new ProjectHead(version.Id, version.Number)
: null;
var record = ProjectRecord.Of(row, mirrored);
ProjectSidecars.WriteRecord(path, record);
Log.Warning("projects", $"Project {row.Id} of sequence {row.SequenceId} had no record in its folder; it is written again from the index");
return record;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The record of project {row.Id} could not be written again from the index; tried again next time", ex);
return null;
}
}
/// <summary>
/// Whether <paramref name="record"/>'s head is one of the versions the index holds below its head (or it names none while
/// the index has one): the record is behind. A record whose head the index does not hold at its number (another chain, a
/// version this Chronos cannot read) is not: it is never written over.
/// </summary>
internal static async Task<bool> BehindAsync(ChronosDatabase db, SequenceProject project, ProjectRecord record, CancellationToken cancellationToken)
{
if (project.HeadVersionId is null) return false;
if (record.Head is not { } head) return true;
return head.Number < project.HeadNumber && await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is { } version
&& version.ProjectId == project.Id && version.Number == head.Number;
}
/// <summary>
/// The time a record written now from the index says it was updated: now, or just after <paramref name="previous"/> (the
/// time the record it replaces says) when a clock behind the one that wrote it would go back. A record's time never goes
/// back, so the newest write of a project (a rename on another PC) is always seen as newer than the one before.
/// </summary>
internal static DateTimeOffset UpdatedAfter(DateTimeOffset previous)
{
var now = DateTimeOffset.UtcNow;
return now > previous ? now : previous.AddMilliseconds(1);
}
/// <summary>The version files of a project numbered above <paramref name="after"/> written from the index (immutable: one there already is left).</summary>
internal static async Task WriteVersionsAsync(ChronosDatabase db, SequencePaths folder, Guid projectId, int after, CancellationToken cancellationToken)
{
while (true)
{
var page = await db.Projects.ListVersionsAfterAsync(projectId, after, WriteBackPage, cancellationToken).ConfigureAwait(false);
foreach (var version in page) ProjectSidecars.WriteVersion(folder.VersionPath(projectId, version.Number, version.Id), version);
if (page.Count < WriteBackPage) return;
after = page[^1].Number;
}
}
/// <summary>
/// <paramref name="name"/>, or "<paramref name="name"/> (2)", "(3)"… (within <see cref="ProjectService.MaxNameLength"/>):
/// the first no project of <paramref name="live"/> but <paramref name="projectId"/> has.
/// </summary>
internal static string FreeName(IReadOnlyList<SequenceProject> live, string name, Guid? projectId)
{
var taken = live.Where(p => p.Id != projectId).Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
var candidate = name;
for (var n = 2; taken.Contains(candidate); n++)
{
var suffix = $" ({n.ToString(CultureInfo.InvariantCulture)})";
candidate = (name.Length + suffix.Length > ProjectService.MaxNameLength ? name[..(ProjectService.MaxNameLength - suffix.Length)].TrimEnd() : name) + suffix;
}
return candidate;
}
/// <summary>
/// The project folders of a sequence: those in the projects' trash (<c>&lt;id&gt;-&lt;time&gt;</c>), then the live ones.
/// The trash comes first so a deletion wins over what is left of the project's live folder (a sync tool moving it into
/// the trash file by file, another PC's delete arriving): a deletion is never undone by a reconcile.
/// </summary>
private static IEnumerable<(string Directory, bool Trashed)> ProjectFolders(SequencePaths folder)
{
if (Directory.Exists(folder.ProjectsTrashDirectory))
{
foreach (var directory in Directory.EnumerateDirectories(folder.ProjectsTrashDirectory).Order(StringComparer.Ordinal))
{
if (File.Exists(Path.Combine(directory, ArchiveLayout.ProjectFile))) yield return (directory, true);
}
}
foreach (var directory in Directory.EnumerateDirectories(folder.ProjectsDirectory).Order(StringComparer.Ordinal))
{
if (Guid.TryParseExact(Path.GetFileName(directory), "N", out _)) yield return (directory, false);
}
}
/// <summary>Whether the index holds project <paramref name="projectId"/> live, of this sequence.</summary>
private async Task<bool> LiveHereAsync(Guid projectId, Guid sequenceId, CancellationToken cancellationToken) =>
await db.Projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) is { DeletedAt: null } row && row.SequenceId == sequenceId;
/// <summary>
/// Whether the live folder <paramref name="directory"/> of project <paramref name="projectId"/>, which has no record, may
/// have its record written again from the index now. Never while the projects' trash holds a folder of it that may be a
/// deletion on its way (<see cref="DeletionOnItsWay"/>), and only once the folder has looked the same, file by file, on
/// two reconciles <see cref="RecordSettle"/> apart: a sync tool still bringing the real record (with a rename, a newer
/// head) has it win. Until then the project is held back.
/// </summary>
private bool Rebuildable(SequencePaths folder, string directory, Guid projectId, Guid sequenceId)
{
var prefix = projectId.ToString("N") + "-";
try
{
if (Directory.Exists(folder.ProjectsTrashDirectory)
&& Directory.EnumerateDirectories(folder.ProjectsTrashDirectory)
.Any(d => Path.GetFileName(d).StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && DeletionOnItsWay(d, projectId, sequenceId, DateTime.UtcNow)))
{
Log.Info("projects", $"Project {projectId} has a folder in the projects' trash; its record is not written again");
return false;
}
var look = Look(directory);
var now = Stopwatch.GetTimestamp();
if (_unsettled.TryGetValue(projectId, out var earlier) && earlier.Look == look)
{
if (Stopwatch.GetElapsedTime(earlier.Since, now) < RecordSettle) return false;
_unsettled.TryRemove(projectId, out _);
return true;
}
_unsettled[projectId] = (sequenceId, look, now);
Log.Info("projects", $"Project {projectId} has no record in its folder; it is written again from the index once the folder settles");
return false;
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The folder of project {projectId} cannot be read now", ex);
return false;
}
}
/// <summary>
/// Whether the folder <paramref name="directory"/> of the projects' trash, named for <paramref name="projectId"/>, may be
/// a deletion of this sequence's project on its way. Its record names the sequence (one of another, a sequence folder
/// copied by hand, is not), or cannot be read (nothing is guessed). Without a record (versions name no sequence) it is
/// one until <see cref="TrashRecordGrace"/> has passed since anything was last written in it, unless a version in it is
/// another project's: a leftover then, which no longer holds a live project back.
/// </summary>
internal static bool DeletionOnItsWay(string directory, Guid projectId, Guid sequenceId, DateTime nowUtc)
{
var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile);
if (File.Exists(recordPath))
{
try
{
return ProjectSidecars.ReadRecord(recordPath).SequenceId == sequenceId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
return true;
}
}
var info = new DirectoryInfo(directory);
var newest = info.LastWriteTimeUtc;
foreach (var entry in info.EnumerateFileSystemInfos("*", SearchOption.AllDirectories))
{
if (entry.LastWriteTimeUtc > newest) newest = entry.LastWriteTimeUtc;
}
if (nowUtc - newest >= TrashRecordGrace) return false;
foreach (var (_, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder)))
{
try
{
if (ProjectSidecars.ReadVersion(path).ProjectId != projectId) return false;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// Not read: proves nothing either way.
}
}
return true;
}
/// <summary>
/// The version files the live folder <paramref name="directory"/> of a deleted project holds that its folder in the
/// projects' trash (<paramref name="deletion"/>) lacks, moved there beside the deleted record: a version pushed into the
/// live folder before the deletion reached this PC is kept with the project, never deleted nor left where nothing reads
/// it. A file the trash has under the same name is left as it is.
/// </summary>
private static void KeepWithDeletion(string directory, string deletion, Guid projectId)
{
var target = Path.Combine(deletion, ArchiveLayout.ProjectVersionsFolder);
foreach (var (number, _, path) in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder)))
{
var to = Path.Combine(target, Path.GetFileName(path));
try
{
if (File.Exists(to) || ProjectSidecars.ReadVersion(path).ProjectId != projectId) continue;
Directory.CreateDirectory(target);
File.Move(path, to);
Log.Warning("projects", $"Version {number} of deleted project {projectId} was in its live folder; it is kept beside its deleted record in {deletion}");
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {path} of deleted project {projectId} could not be moved beside its deleted record; tried again next time", ex);
}
}
}
/// <summary>What a project folder holds now, file by file (path, size, time written), as one hash.</summary>
private static string Look(string directory)
{
var text = new StringBuilder();
foreach (var file in new DirectoryInfo(directory).EnumerateFiles("*", SearchOption.AllDirectories).OrderBy(f => f.FullName, StringComparer.Ordinal))
{
text.Append(Path.GetRelativePath(directory, file.FullName)).Append('|').Append(file.Length.ToString(CultureInfo.InvariantCulture)).Append('|')
.Append(file.LastWriteTimeUtc.Ticks.ToString(CultureInfo.InvariantCulture)).Append('\n');
}
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text.ToString())));
}
/// <summary>
/// The record with the version files that follow its head adopted (a push that wrote its version but not the record),
/// and the files past the head that do not follow it moved aside; the record is written again when its head moved. A
/// file past the head that cannot be read proves nothing: then none is moved aside.
/// </summary>
private static ProjectRecord RollForward(SequencePaths folder, ProjectRecord record, string directory, bool writable)
{
var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
var files = ProjectSidecars.ListVersions(versions);
var head = record.Head;
var moved = false;
var unread = false;
while (true)
{
var next = (head?.Number ?? 0) + 1;
ProjectVersion? follower = null;
foreach (var candidate in files.Where(f => f.Number == next))
{
try
{
var version = ProjectSidecars.ReadVersion(candidate.Path);
if (version.ProjectId == record.Id && version.Id == candidate.Id && version.ParentId == head?.Id)
{
follower = version;
break;
}
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {candidate.Path} cannot be read", ex);
unread = true;
}
}
if (follower is null) break;
head = new ProjectHead(follower.Id, follower.Number);
moved = true;
}
if (writable && !unread)
{
foreach (var stray in files.Where(f => f.Number > (head?.Number ?? 0)))
SetAside(versions, stray.Path, $"does not follow the head of project {record.Id}");
}
if (!moved) return record;
var rolled = record with { Head = head, Updated = UpdatedAfter(record.Updated) };
if (!writable) return rolled;
try
{
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(record.Id), rolled);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// The index takes the versions all the same; the record is written with them before anything else is (ProjectService).
Log.Warning("projects", $"The record of project {record.Id} could not be moved forward to version {head!.Number}", ex);
}
return rolled;
}
/// <summary>A version file moved aside into <c>versions/.stray</c> (kept, never deleted).</summary>
private static void SetAside(string versions, string path, string why)
{
try
{
var aside = Path.Combine(versions, StrayFolder);
Directory.CreateDirectory(aside);
File.Move(path, Path.Combine(aside, Path.GetFileName(path) + "." + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff", CultureInfo.InvariantCulture)));
Log.Warning("projects", $"Version file {path} {why}; moved aside");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {path} could not be moved aside", ex);
}
}
/// <summary>
/// Writes what the index lacks of one project's record and versions. False when its head could not be reached: the
/// record names a version this Chronos cannot read (damaged, of a newer Chronos), and the index's head stays on the
/// newest one it holds below it.
/// </summary>
private async Task<bool> IndexAsync(ProjectRecord record, string directory, bool trashed, bool writable, CancellationToken cancellationToken)
{
var row = await db.Projects.GetAsync(record.Id, cancellationToken).ConfigureAwait(false);
if (row is null)
{
var project = record.ToProject();
if (project.DeletedAt is null)
{
var live = await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
if (project.IsMain && live.Any(p => p.IsMain))
{
Log.Warning("projects", $"Project {record.Id} says it is Main, but sequence {record.SequenceId} has one; indexed as another project");
project = project with { IsMain = false };
}
project = project with { Name = FreeName(live, project.Name, project.Id) };
}
// The head follows its versions below, so a version the index cannot take never leaves a head without its row.
row = project with { HeadVersionId = null, HeadNumber = 0 };
await db.Projects.InsertAsync(row, cancellationToken).ConfigureAwait(false);
}
// A record behind the index (a projects folder put back from an older backup: its head is a version the index holds
// below its own, or it was written before what the index knows of the project, as a rename made since) says what was:
// its name is not taken, and the heal writes the index's back (ProjectService). A record on another chain is not behind.
var stale = await BehindAsync(db, row, record, cancellationToken).ConfigureAwait(false) || record.Updated < row.UpdatedAt;
await IndexVersionsAsync(record, directory, writable, cancellationToken).ConfigureAwait(false);
var (headId, headNumber) = (row.HeadVersionId, row.HeadNumber);
var reached = true;
if (record.Head is { } head && (head.Number > headNumber || (head.Number == headNumber && head.Id != headId)))
{
if (await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.ProjectId == record.Id
&& indexed.Number == head.Number)
{
(headId, headNumber) = (head.Id, head.Number);
}
else
{
reached = false;
Log.Warning("projects", $"The newest version of project {record.Id} ({head.Number}) cannot be read; the project is shown as far as it can be, and not changed");
var below = await db.Projects.ListVersionsAsync(record.Id, head.Number, 1, cancellationToken).ConfigureAwait(false);
if (below.Count > 0 && below[0].Number > headNumber) (headId, headNumber) = (below[0].Id, below[0].Number);
}
}
var updated = row with { HeadVersionId = headId, HeadNumber = headNumber };
if (record.Deleted is not null || trashed)
{
updated = updated with { DeletedAt = row.DeletedAt ?? record.Deleted };
}
else if (row.DeletedAt is not null || (row.Name != record.Name && !stale))
{
var others = (await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
.Where(p => p.Id != record.Id).ToList();
if (row.DeletedAt is not null)
{
// Its live record says it is not deleted: listed again, Main again unless the sequence has another.
Log.Warning("projects", $"Project {record.Id} of sequence {record.SequenceId} is live in its sidecars; it is listed again");
updated = updated with { DeletedAt = null, IsMain = row.IsMain && !others.Any(p => p.IsMain) };
}
// The record's name (unless it is behind), else (another project has it) the name the index has, else one free.
updated = updated with
{
Name = !stale && FreeName(others, record.Name, record.Id) == record.Name ? record.Name
: FreeName(others, row.Name, record.Id) == row.Name ? row.Name
: FreeName(others, stale ? row.Name : record.Name, record.Id),
};
// What the index knows of the project is now as new as the record, so an older one put back later is seen as such.
if (!stale && record.Updated > updated.UpdatedAt) updated = updated with { UpdatedAt = record.Updated };
}
if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false);
return reached || record.Deleted is not null || trashed;
}
/// <summary>
/// The version files up to the record's head the index lacks, taken in. Of several files of one number, the one the
/// record's chain names is taken and the others are moved aside; when the chain cannot tell (a version on it cannot be
/// read), none is taken.
/// </summary>
private async Task IndexVersionsAsync(ProjectRecord record, string directory, bool writable, CancellationToken cancellationToken)
{
var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
var files = ProjectSidecars.ListVersions(versions);
var limit = record.Head?.Number ?? 0;
var indexed = new Dictionary<int, Guid>(await db.Projects.ListVersionIdsAsync(record.Id, cancellationToken).ConfigureAwait(false));
Dictionary<int, Guid>? chain = null;
foreach (var number in files.Where(f => f.Number <= limit && !indexed.ContainsKey(f.Number)).GroupBy(f => f.Number))
{
var candidates = number.ToList();
if (candidates.Count > 1)
{
chain ??= Chain(record, files);
var named = candidates.Where(c => chain.TryGetValue(c.Number, out var id) && id == c.Id).ToList();
if (named.Count == 0)
{
Log.Warning("projects", $"Project {record.Id} has {candidates.Count} version files numbered {number.Key} and its record's chain names none; none is indexed");
continue;
}
if (writable)
{
foreach (var other in candidates.Where(c => c.Id != named[0].Id))
SetAside(versions, other.Path, $"has the number of another version of project {record.Id}");
}
candidates = named;
}
foreach (var file in candidates)
{
ProjectVersion version;
try
{
version = ProjectSidecars.ReadVersion(file.Path);
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {file.Path} cannot be read; it is not indexed", ex);
continue;
}
if (version.ProjectId != record.Id || version.Number != file.Number || version.Id != file.Id) continue;
if (await db.Projects.GetVersionAsync(version.Id, cancellationToken).ConfigureAwait(false) is not null)
{
Log.Warning("projects", $"Version file {file.Path} holds a version the index has under another number; it is not indexed");
continue;
}
await db.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
indexed[version.Number] = version.Id;
}
}
}
/// <summary>The version ids by number along the record's chain, from its head back through each parent, as far as their files read.</summary>
private static Dictionary<int, Guid> Chain(ProjectRecord record, IReadOnlyList<(int Number, Guid Id, string Path)> files)
{
var chain = new Dictionary<int, Guid>();
var byId = files.GroupBy(f => f.Id).ToDictionary(g => g.Key, g => g.First());
for (var id = record.Head?.Id; id is { } current && byId.TryGetValue(current, out var file) && !chain.ContainsKey(file.Number);)
{
chain[file.Number] = file.Id;
try
{
id = ProjectSidecars.ReadVersion(file.Path).ParentId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {file.Path} cannot be read", ex);
break;
}
}
return chain;
}
}