fix(projects): one atomic catch-up merge, Main settled from the records, lost records rebuilt
The catch-up merge of a push whose mirror a crash cut short with settings written meanwhile was two appends (External, then Merge), resumed by a purely structural test that also fired on ordinary histories (two outside edits in a row, a second index with an older mirror) and reverted the latest edit. It is now one Merge version on the head in which the settings written meanwhile (the settings in force, which older clients wrote) win where both changed; the head's values stay in its version. A merge that adds nothing to the head makes no version, so a cut before its mirror is finished next time by the same index or a rebuilt one without any marker. The structural resume is gone. An index meeting Main with a lagging record merges project.json the same way, so a newer outside edit is no longer reverted by an older push. Versions the catch-up records as mirrored are named in Main's record too when project.json holds them, so records no longer lag behind externals. Duplicate Main claimants (an index that met the sequence before Main's record arrived) are settled from the records: the oldest claimant with a version is Main in every index, the others are ordinary projects, and their records say so when the archive takes writes. A live project folder whose record is missing while the index holds the project gets its record (and missing version files) written again from the index instead of being held back forever. A record behind the index no longer gives the index its old name. MirrorAsync reads the version leniently and skips a newer one, as the other branches do. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
488b588ad9
commit
bc87a57830
5 files changed
+398
-130
No files matched your search
@@ -24,7 +24,8 @@ internal sealed record ProjectReconcile(bool Missing, IReadOnlySet<Guid> Seen, I
|
||||
/// behind on moves forward, a name or deletion recorded since is taken, and a row the index marked deleted is listed again
|
||||
/// when its live record says it is not. Only a record that says so, live or in the projects' trash, deletes a project:
|
||||
/// never a folder that is not there (moved for a moment, a backup put back, a sync tool) nor a record that cannot be read,
|
||||
/// whose project stays as the index has it (<see cref="ProjectReconcile.HeldBack"/>). A version file written past its
|
||||
/// whose project stays as the index has it (<see cref="ProjectReconcile.HeldBack"/>); a record missing from the folder of a
|
||||
/// live project the index holds is written again from the index when the archive takes writes. A version file written past its
|
||||
/// record's head by a push a crash cut short is adopted when it follows the head (its file is complete: versions are
|
||||
/// written atomically, then the record); any other file past the head, or a second file of a number the record's chain
|
||||
/// names another for, is moved aside to <c>versions/.stray</c>, never deleted. Runs under the sequence's lease, or without
|
||||
@@ -34,6 +35,9 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
|
||||
{
|
||||
private const string StrayFolder = ".stray";
|
||||
|
||||
/// <summary>Versions read at once when a project's sidecars are written again from the index.</summary>
|
||||
private const int WriteBackPage = 500;
|
||||
|
||||
/// <summary>Reconciles the projects of <paramref name="sequenceId"/> in <paramref name="folder"/>; the caller holds the sequence's lease.</summary>
|
||||
/// <param name="writable">False for a read-only archive: the sidecars are read as they are and only the index is written
|
||||
/// (a version past its record's head is indexed, not adopted into the record; nothing is moved aside).</param>
|
||||
@@ -46,29 +50,39 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
|
||||
foreach (var (directory, trashed) in ProjectFolders(folder))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
ProjectRecord record;
|
||||
ProjectRecord? record = null;
|
||||
var recordPath = Path.Combine(directory, ArchiveLayout.ProjectFile);
|
||||
try
|
||||
{
|
||||
record = ProjectSidecars.ReadRecord(Path.Combine(directory, ArchiveLayout.ProjectFile));
|
||||
record = ProjectSidecars.ReadRecord(recordPath);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
// Its folder names it (<id>, or <id>-<time> in the trash).
|
||||
var name = Path.GetFileName(directory);
|
||||
if (!Guid.TryParseExact(trashed && name.Length > 32 ? name[..32] : name, "N", out var named)) continue;
|
||||
if (!trashed && !File.Exists(Path.Combine(directory, ArchiveLayout.ProjectFile))
|
||||
&& await db.Projects.GetAsync(named, cancellationToken).ConfigureAwait(false) is null)
|
||||
if (!trashed && !File.Exists(recordPath))
|
||||
{
|
||||
// Versions with no record, of a project no index row names: a creation a crash or a lost disk cut short before its
|
||||
// record (versions are written first), or one a sync tool is still bringing. No project of the sequence is
|
||||
// hidden in it, so it holds nothing back; it is left as it is, and indexed once its record is there.
|
||||
Log.Info("projects", $"{directory} holds no project record; it is left as it is");
|
||||
var row = await db.Projects.GetAsync(named, cancellationToken).ConfigureAwait(false);
|
||||
if (row is null)
|
||||
{
|
||||
// Versions with no record, of a project no index row names: a creation a crash or a lost disk cut short before
|
||||
// its record (versions are written first), or one a sync tool is still bringing. No project of the sequence is
|
||||
// hidden in it, so it holds nothing back; it is left as it is, and indexed once its record is there.
|
||||
Log.Info("projects", $"{directory} holds no project record; it is left as it is");
|
||||
continue;
|
||||
}
|
||||
// A record lost (a sync tool, a disk) of a live project the index holds: written again from the index.
|
||||
if (writable && row is { DeletedAt: null } && row.SequenceId == sequenceId)
|
||||
record = await RecordFromIndexAsync(folder, row, recordPath, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (record is null)
|
||||
{
|
||||
Log.Warning("projects", $"The project in {directory} cannot be read; it stays as the index has it", ex);
|
||||
// Its row stays as it is, never taken for gone nor written again, and a live one is held back (it may be Main).
|
||||
if (seen.Add(named) && !trashed) heldBack.Add(named);
|
||||
continue;
|
||||
}
|
||||
Log.Warning("projects", $"The project in {directory} cannot be read; it stays as the index has it", ex);
|
||||
// Its row stays as it is, never taken for gone nor written again, and a live one is held back (it may be Main).
|
||||
if (seen.Add(named) && !trashed) heldBack.Add(named);
|
||||
continue;
|
||||
}
|
||||
// A sequence folder copied by hand carries the projects of its source, which stay that sequence's.
|
||||
if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue;
|
||||
@@ -91,7 +105,7 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
|
||||
}
|
||||
try
|
||||
{
|
||||
await SettleMainAsync(sequenceId, live, heldBack, cancellationToken).ConfigureAwait(false);
|
||||
await SettleMainAsync(folder, sequenceId, live, heldBack, writable, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (DatabaseException ex) when (ex.IsUniqueViolation || ex.IsForeignKeyViolation)
|
||||
{
|
||||
@@ -115,33 +129,98 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The index's Main taken from the records, which say which project is Main (another index of the archive passed it on:
|
||||
/// an index-only Main written back beside the one made into the archive meanwhile): when the index's Main is read and its
|
||||
/// record no longer says it is Main, or the index has none, the oldest project whose record says it is becomes Main, and
|
||||
/// both take their records' names where free. Only records in step with the index count (a record behind it, put back
|
||||
/// from an older backup, says what was): never while the index's Main is held back or its record is not in step, and
|
||||
/// never more than one Main.
|
||||
/// The index's Main taken from the records, which say which project is Main. When several records in step say so (an
|
||||
/// index that met the sequence while Main's record was not there yet, a sync tool still bringing it, made a Main of its
|
||||
/// own; another index passed Main on), the oldest of them that has a version (by the time its record says it was made,
|
||||
/// then by id) is Main, the same in every index, and the others are ordinary projects, every version kept: in the index,
|
||||
/// and in their records too when the archive takes writes, so each index and older Chronos reads them alike. Main and the
|
||||
/// project it passes from take their records' names where free. Only records in step with the index count (a record
|
||||
/// behind it, put back from an older backup, says what was): never while the index's Main is held back or its record is
|
||||
/// not in step, and never more than one Main.
|
||||
/// </summary>
|
||||
private async Task SettleMainAsync(Guid sequenceId, IReadOnlyDictionary<Guid, ProjectRecord> live, IReadOnlySet<Guid> heldBack, CancellationToken cancellationToken)
|
||||
private async Task SettleMainAsync(SequencePaths folder, Guid sequenceId, Dictionary<Guid, ProjectRecord> live, IReadOnlySet<Guid> heldBack, bool writable,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var rows = await db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
|
||||
ProjectRecord? InStep(SequenceProject row) =>
|
||||
!heldBack.Contains(row.Id) && live.TryGetValue(row.Id, out var record) && record.Head?.Id == row.HeadVersionId ? record : null;
|
||||
var main = rows.FirstOrDefault(p => p.IsMain);
|
||||
var own = main is null ? null : InStep(main);
|
||||
if (main is not null && own is not { IsMain: false }) return;
|
||||
var claimant = rows.Where(p => !p.IsMain && InStep(p) is { IsMain: true }).OrderBy(p => p.CreatedAt).ThenBy(p => p.Id).FirstOrDefault();
|
||||
if (claimant is null) return;
|
||||
var others = rows.Where(p => p.Id != claimant.Id && p.Id != main?.Id).ToList();
|
||||
var promoted = claimant with { IsMain = true, Name = FreeName(others, live[claimant.Id].Name, claimant.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
|
||||
Log.Warning("projects", $"The records of sequence {sequenceId} name project {claimant.Id} as Main{(main is null ? string.Empty : $", not {main.Id}")}; the index takes it");
|
||||
if (main is null)
|
||||
if (main is not null && own is null) return;
|
||||
var claimants = rows.Where(p => p.HeadVersionId is not null && InStep(p) is { IsMain: true })
|
||||
.OrderBy(p => live[p.Id].Created.UtcTicks).ThenBy(p => p.Id).ToList();
|
||||
if (claimants.Count == 0) return;
|
||||
var winner = claimants[0];
|
||||
if (winner.Id != main?.Id)
|
||||
{
|
||||
await db.Projects.UpdateAsync(promoted, cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
var others = rows.Where(p => p.Id != winner.Id && p.Id != main?.Id).ToList();
|
||||
var promoted = winner with { IsMain = true, Name = FreeName(others, live[winner.Id].Name, winner.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
|
||||
Log.Warning("projects", $"The records of sequence {sequenceId} name project {winner.Id} as Main{(main is null ? string.Empty : $", not {main.Id}")}; the index takes it");
|
||||
if (main is null)
|
||||
{
|
||||
await db.Projects.UpdateAsync(promoted, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
var demoted = main with { IsMain = false, Name = FreeName([.. others, promoted], own!.Name, main.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
|
||||
await SwapMainAsync(db, demoted, promoted, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
if (!writable || claimants.Count < 2) return;
|
||||
// The records that say Main and are not: ordinary projects, as the index names them.
|
||||
foreach (var loser in claimants.Skip(1))
|
||||
{
|
||||
if (await db.Projects.GetAsync(loser.Id, cancellationToken).ConfigureAwait(false) is not { IsMain: false, DeletedAt: null } row) continue;
|
||||
var record = live[loser.Id] with { IsMain = false, Name = row.Name, Mirrored = null, Updated = DateTimeOffset.UtcNow };
|
||||
try
|
||||
{
|
||||
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(loser.Id), record);
|
||||
live[loser.Id] = record;
|
||||
Log.Warning("projects", $"The record of project {loser.Id} of sequence {sequenceId} said it was Main beside project {winner.Id}; it says it is '{row.Name}'");
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("projects", $"The record of project {loser.Id} still says it is Main; tried again next time", ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The record of a live project the index holds, written again from the index when its folder has none (a sync tool, a
|
||||
/// disk lost it): the version files the folder lacks first (those there are left as they are), then the record naming
|
||||
/// the index's head and, for Main, the version this index mirrored. The versions the folder holds past that head are
|
||||
/// then adopted as after any push a crash cut short. Null when it cannot be written now (tried again next time).
|
||||
/// </summary>
|
||||
private async Task<ProjectRecord?> RecordFromIndexAsync(SequencePaths folder, SequenceProject row, string path, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await WriteVersionsAsync(db, folder, row.Id, 0, cancellationToken).ConfigureAwait(false);
|
||||
var mirrored = row.IsMain && row.MirroredVersionId is { } id && await db.Projects.GetVersionAsync(id, cancellationToken).ConfigureAwait(false) is { } version
|
||||
? new ProjectHead(version.Id, version.Number)
|
||||
: null;
|
||||
var record = ProjectRecord.Of(row, mirrored);
|
||||
ProjectSidecars.WriteRecord(path, record);
|
||||
Log.Warning("projects", $"Project {row.Id} of sequence {row.SequenceId} had no record in its folder; it is written again from the index");
|
||||
return record;
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
|
||||
{
|
||||
Log.Warning("projects", $"The record of project {row.Id} could not be written again from the index; tried again next time", ex);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The version files of a project numbered above <paramref name="after"/> written from the index (immutable: one there already is left).</summary>
|
||||
internal static async Task WriteVersionsAsync(ChronosDatabase db, SequencePaths folder, Guid projectId, int after, CancellationToken cancellationToken)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var page = await db.Projects.ListVersionsAfterAsync(projectId, after, WriteBackPage, cancellationToken).ConfigureAwait(false);
|
||||
foreach (var version in page) ProjectSidecars.WriteVersion(folder.VersionPath(projectId, version.Number, version.Id), version);
|
||||
if (page.Count < WriteBackPage) return;
|
||||
after = page[^1].Number;
|
||||
}
|
||||
var demoted = main with { IsMain = false, Name = FreeName([.. others, promoted], own!.Name, main.Id), MirroredVersionId = null, MirroredSettingsVersion = null };
|
||||
await SwapMainAsync(db, demoted, promoted, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -272,6 +351,8 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
|
||||
row = project with { HeadVersionId = null, HeadNumber = 0 };
|
||||
await db.Projects.InsertAsync(row, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
// A record behind the index (a projects folder put back from an older backup) says what was: its name is not taken.
|
||||
var behind = row.HeadVersionId is not null && (record.Head?.Number ?? 0) < row.HeadNumber;
|
||||
await IndexVersionsAsync(record, directory, writable, cancellationToken).ConfigureAwait(false);
|
||||
var (headId, headNumber) = (row.HeadVersionId, row.HeadNumber);
|
||||
var reached = true;
|
||||
@@ -295,7 +376,7 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
|
||||
{
|
||||
updated = updated with { DeletedAt = row.DeletedAt ?? record.Deleted };
|
||||
}
|
||||
else if (row.DeletedAt is not null || row.Name != record.Name)
|
||||
else if (row.DeletedAt is not null || (row.Name != record.Name && !behind))
|
||||
{
|
||||
var others = (await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
|
||||
.Where(p => p.Id != record.Id).ToList();
|
||||
@@ -305,12 +386,12 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
|
||||
Log.Warning("projects", $"Project {record.Id} of sequence {record.SequenceId} is live in its sidecars; it is listed again");
|
||||
updated = updated with { DeletedAt = null, IsMain = row.IsMain && !others.Any(p => p.IsMain) };
|
||||
}
|
||||
// The record's name, else (another project has it) the name the index has, else one free.
|
||||
// The record's name (unless it is behind), else (another project has it) the name the index has, else one free.
|
||||
updated = updated with
|
||||
{
|
||||
Name = FreeName(others, record.Name, record.Id) == record.Name ? record.Name
|
||||
Name = !behind && FreeName(others, record.Name, record.Id) == record.Name ? record.Name
|
||||
: FreeName(others, row.Name, record.Id) == row.Name ? row.Name
|
||||
: FreeName(others, record.Name, record.Id),
|
||||
: FreeName(others, behind ? row.Name : record.Name, record.Id),
|
||||
};
|
||||
}
|
||||
if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
Reference in new issue
Block a user