fix(projects): sidecars stay the truth when they cannot be read, are missing or newer

A project record that could not be read once, or a missing .chronos/projects, marked
projects deleted for good and made a second Main; an index meeting Main with no mirror
state took its own stale settings column as an outside change and reverted the pushed
head; a newer-format or unreadable head made every project call fail or rewrote the
record's head over the real history. Now:

- Deletion is never inferred from absence or a read failure: an unreadable record holds
  its project back (listed as indexed, nothing written for it, no second Main), a row
  marked deleted is revived when its live record says so, and projects the archive has
  no folder for are written back from the index.
- Appends re-read the record under the lease and refuse when its head is not the
  index's; Main is only made when the sequence has none anywhere.
- An index with no mirror state decides from project.json and the record's new
  mirrored member (older records: the head's ancestry), never from its own column.
- A newer-format Main head is listed and pulled but never mirrored, merged or built on;
  pushes must parse strictly and fit the one 64 MB version limit (TooLarge otherwise).
- Duplicate version numbers and name/Main clashes are contained per project, so a scan
  or rebuild never stops.
- Delete moves the folder first, then writes the deleted record in the trash.
- The home's read-back for workspace pruning and Remove local copy reads the version
  files (IProjectHome.HoldsAsync), not the index.
- A read-only archive is read without writing: its projects open from the sidecars and
  a sequence with no Main gets one in the index only, written back once writable.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alby96andClaude Opus 5.5 committed 2026-11-03 10:13:40 +01:00
1 parent 50e28fd63d
commit d940fc7c61
6 files changed
+1247 -158

No files matched your search

+7 -5
View File
@@ -491,10 +491,8 @@ public sealed class WorkspaceHost
var removed = 0;
foreach (var version in stored.Where(v => v.Id != basis?.VersionId && !kept.Contains(v.Id)))
{
if (await home.GetVersionAsync(projectId, version.Id, cancellationToken).ConfigureAwait(false) is not { } atHome
|| !string.Equals(atHome.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase)
|| !ProjectContent.Verifies(atHome.SettingsJson, atHome.ContentHash))
continue;
// Read back from where the home keeps it, not from an index of it.
if (!await home.HoldsAsync(projectId, version, cancellationToken).ConfigureAwait(false)) continue;
File.Delete(project.VersionPath(version));
removed++;
}
@@ -528,16 +526,20 @@ public sealed class WorkspaceHost
var home = HomeOf(workspace.Home) ?? throw new WorkspaceException(Loc.L("Errors.Workspace.Offline"));
ProjectVersion head;
ProjectVersion? basis;
bool held;
try
{
head = await home.HeadAsync(projectId, cancellationToken).ConfigureAwait(false);
basis = workspace.Base is { } b ? await home.GetVersionAsync(projectId, b.VersionId, cancellationToken).ConfigureAwait(false) : head;
// Read back from where the home keeps them, not from an index of them.
held = basis is not null && await home.HoldsAsync(projectId, head, cancellationToken).ConfigureAwait(false)
&& (basis.Id == head.Id || await home.HoldsAsync(projectId, basis, cancellationToken).ConfigureAwait(false));
}
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or DatabaseException)
{
throw new WorkspaceException(Loc.L("Errors.Workspace.Offline"), ex);
}
if (!ProjectContent.Verifies(head.SettingsJson, head.ContentHash) || basis is null || !ProjectContent.Verifies(basis.SettingsJson, basis.ContentHash)
if (!held || !ProjectContent.Verifies(head.SettingsJson, head.ContentHash) || basis is null || !ProjectContent.Verifies(basis.SettingsJson, basis.ContentHash)
|| (workspace.Base is { } known && !string.Equals(known.ContentHash, basis.ContentHash, StringComparison.OrdinalIgnoreCase)))
throw new WorkspaceException(Loc.L("Errors.Workspace.HomeCheck"));
}
+27
View File
@@ -33,6 +33,18 @@ public interface IProjectHome
/// <summary>A version of the project by id; null when the home has none.</summary>
Task<ProjectVersion?> GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken);
/// <summary>
/// Whether the home holds <paramref name="version"/> durably, read back from where it keeps it and matching its content
/// hash: what a workspace asks before it drops its own copy. A home that keeps no other copy answers from
/// <see cref="GetVersionAsync"/>.
/// </summary>
async Task<bool> HoldsAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(version);
return await GetVersionAsync(projectId, version.Id, cancellationToken).ConfigureAwait(false) is { } held
&& string.Equals(held.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase) && ProjectContent.Verifies(held.SettingsJson, held.ContentHash);
}
/// <summary>The versions numbered above <paramref name="afterNumber"/>, oldest first, at most <paramref name="limit"/>.</summary>
Task<IReadOnlyList<ProjectVersion>> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken);
@@ -91,6 +103,16 @@ public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projec
return version is not null && version.ProjectId == projectId ? version : null;
});
/// <summary>Read back from the version's file in the archive, never from the index's row (<see cref="ProjectService.StoredAsync"/>).</summary>
public Task<bool> HoldsAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken) =>
Reach(async () =>
{
ArgumentNullException.ThrowIfNull(version);
var project = await Own(projectId, cancellationToken).ConfigureAwait(false);
return project.DeletedAt is null && ProjectContent.Verifies(version.SettingsJson, version.ContentHash)
&& await projects.StoredAsync(projectId, version, cancellationToken).ConfigureAwait(false);
});
public Task<IReadOnlyList<ProjectVersion>> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) =>
Reach(async () =>
{
@@ -151,5 +173,10 @@ public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projec
// Another index of the archive moved the sequence: this one finds it at its next scan.
throw new HomeUnavailableException(ex.Message, ex);
}
catch (ProjectsUnreadableException ex)
{
// A record or a version this Chronos cannot read now (damaged, held open, of a newer Chronos): nothing was written.
throw new HomeUnavailableException(ex.Message, ex);
}
}
}
+236 -82
View File
@@ -5,50 +5,97 @@ using Chronos.Core.Model;
namespace Chronos.Data.Projects;
/// <summary>What a reconcile of a sequence's projects met (<see cref="ProjectIndexer.ReconcileAsync"/>).</summary>
/// <param name="Missing">The sequence has no projects folder: nothing was read, and nothing taken for gone.</param>
/// <param name="Seen">The projects the archive has a folder for, live or in the projects' trash, whether or not its record read.</param>
/// <param name="HeldBack">The live projects the index could not be brought in line with: a record that cannot be read (damaged,
/// of a newer Chronos, a read failure), a head whose version this Chronos cannot read, or a row the index refused. Each stays
/// as the index has it, and nothing is written for it until it reads again.</param>
/// <param name="Live">The records of the live projects read, by id (their head as moved forward).</param>
internal sealed record ProjectReconcile(bool Missing, IReadOnlySet<Guid> Seen, IReadOnlySet<Guid> HeldBack, IReadOnlyDictionary<Guid, ProjectRecord> Live)
{
/// <summary>Every live project of the folder was read and indexed: a sequence without a Main may be given one.</summary>
public bool Complete => HeldBack.Count == 0;
}
/// <summary>
/// Brings the index's projects of one sequence in line with their sidecars (owner review 2, R8), which are the truth: a
/// project or version the index lacks is added (an index rebuilt, a sequence restored from the trash), a head the index is
/// behind on moves forward, a name, Main flag or deletion recorded since is taken, and a live index row whose folder is gone
/// is marked deleted. A version file written past its record's head by a push a crash cut short is adopted when it
/// follows the head (its file is complete: versions are written atomically, then the record); any other file past the
/// head is moved aside to <c>versions/.stray</c>, never deleted. Runs under the sequence's lease.
/// behind on moves forward, a name or deletion recorded since is taken, and a row the index marked deleted is listed again
/// when its live record says it is not. Only a record that says so, live or in the projects' trash, deletes a project:
/// never a folder that is not there (moved for a moment, a backup put back, a sync tool) nor a record that cannot be read,
/// whose project stays as the index has it (<see cref="ProjectReconcile.HeldBack"/>). A version file written past its
/// record's head by a push a crash cut short is adopted when it follows the head (its file is complete: versions are
/// written atomically, then the record); any other file past the head, or a second file of a number the record's chain
/// names another for, is moved aside to <c>versions/.stray</c>, never deleted. Runs under the sequence's lease, or without
/// writing to the archive at all when it is read-only.
/// </summary>
internal sealed class ProjectIndexer(ChronosDatabase db)
{
private const string StrayFolder = ".stray";
/// <summary>Reconciles the projects of <paramref name="sequenceId"/> in <paramref name="folder"/>; the caller holds the sequence's lease.</summary>
public async Task ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken)
/// <param name="writable">False for a read-only archive: the sidecars are read as they are and only the index is written
/// (a version past its record's head is indexed, not adopted into the record; nothing is moved aside).</param>
public async Task<ProjectReconcile> ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken, bool writable = true)
{
var seen = new HashSet<Guid>();
if (Directory.Exists(folder.ProjectsDirectory))
var heldBack = new HashSet<Guid>();
var live = new Dictionary<Guid, ProjectRecord>();
if (!Directory.Exists(folder.ProjectsDirectory)) return new ProjectReconcile(true, seen, heldBack, live);
foreach (var (directory, trashed) in ProjectFolders(folder))
{
foreach (var (directory, trashed) in ProjectFolders(folder))
cancellationToken.ThrowIfCancellationRequested();
ProjectRecord record;
try
{
cancellationToken.ThrowIfCancellationRequested();
ProjectRecord record;
try
{
record = ProjectSidecars.ReadRecord(Path.Combine(directory, ArchiveLayout.ProjectFile));
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The project in {directory} cannot be read", ex);
continue;
}
// A sequence folder copied by hand carries the projects of its source, which stay that sequence's.
if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue;
if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) };
if (record.Deleted is null) record = RollForward(folder, record, directory);
await IndexAsync(record, directory, cancellationToken).ConfigureAwait(false);
record = ProjectSidecars.ReadRecord(Path.Combine(directory, ArchiveLayout.ProjectFile));
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The project in {directory} cannot be read; it stays as the index has it", ex);
// Its folder names it (<id>, or <id>-<time> in the trash): its row stays as it is, never taken for gone nor written
// again, and a live one is held back (it may be Main).
var name = Path.GetFileName(directory);
if (Guid.TryParseExact(trashed && name.Length > 32 ? name[..32] : name, "N", out var named) && seen.Add(named) && !trashed) heldBack.Add(named);
continue;
}
// A sequence folder copied by hand carries the projects of its source, which stay that sequence's.
if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue;
if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) };
if (record.Deleted is null)
{
record = RollForward(folder, record, directory, writable);
live[record.Id] = record;
}
try
{
if (!await IndexAsync(record, directory, trashed, writable, cancellationToken).ConfigureAwait(false)) heldBack.Add(record.Id);
}
catch (DatabaseException ex) when (ex.IsUniqueViolation || ex.IsForeignKeyViolation)
{
// One project the index refuses never stops the others, nor a scan of the archive: it is read again next time.
Log.Warning("projects", $"Project {record.Id} of sequence {sequenceId} could not be indexed", ex);
if (record.Deleted is null) heldBack.Add(record.Id);
}
}
foreach (var gone in await db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
return new ProjectReconcile(false, seen, heldBack, live);
}
/// <summary>
/// <paramref name="name"/>, or "<paramref name="name"/> (2)", "(3)"… (within <see cref="ProjectService.MaxNameLength"/>):
/// the first no project of <paramref name="live"/> but <paramref name="projectId"/> has.
/// </summary>
internal static string FreeName(IReadOnlyList<SequenceProject> live, string name, Guid? projectId)
{
var taken = live.Where(p => p.Id != projectId).Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
var candidate = name;
for (var n = 2; taken.Contains(candidate); n++)
{
if (seen.Contains(gone.Id)) continue;
Log.Warning("projects", $"Project {gone.Id} of sequence {sequenceId} has no folder any more; it is marked deleted");
await db.Projects.UpdateAsync(gone with { DeletedAt = DateTimeOffset.UtcNow, UpdatedAt = DateTimeOffset.UtcNow }, cancellationToken).ConfigureAwait(false);
var suffix = $" ({n.ToString(CultureInfo.InvariantCulture)})";
candidate = (name.Length + suffix.Length > ProjectService.MaxNameLength ? name[..(ProjectService.MaxNameLength - suffix.Length)].TrimEnd() : name) + suffix;
}
return candidate;
}
/// <summary>The project folders of a sequence: the live ones, then those in the projects' trash (<c>&lt;id&gt;-&lt;time&gt;</c>).</summary>
@@ -67,25 +114,26 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
/// <summary>
/// The record with the version files that follow its head adopted (a push that wrote its version but not the record),
/// and the files past the head that do not follow it moved aside; the record is written again when its head moved.
/// and the files past the head that do not follow it moved aside; the record is written again when its head moved. A
/// file past the head that cannot be read proves nothing: then none is moved aside.
/// </summary>
private static ProjectRecord RollForward(SequencePaths folder, ProjectRecord record, string directory)
private static ProjectRecord RollForward(SequencePaths folder, ProjectRecord record, string directory, bool writable)
{
var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
var files = ProjectSidecars.ListVersions(versions);
var head = record.Head;
var moved = false;
var unread = false;
while (true)
{
var next = (head?.Number ?? 0) + 1;
var candidates = files.Where(f => f.Number == next).ToList();
ProjectVersion? follower = null;
foreach (var candidate in candidates)
foreach (var candidate in files.Where(f => f.Number == next))
{
try
{
var version = ProjectSidecars.ReadVersion(candidate.Path);
if (version.ProjectId == record.Id && version.ParentId == head?.Id)
if (version.ProjectId == record.Id && version.Id == candidate.Id && version.ParentId == head?.Id)
{
follower = version;
break;
@@ -94,86 +142,192 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {candidate.Path} cannot be read", ex);
unread = true;
}
}
if (follower is null) break;
head = new ProjectHead(follower.Id, follower.Number);
moved = true;
}
foreach (var stray in files.Where(f => f.Number > (head?.Number ?? 0)))
if (writable && !unread)
{
try
{
var aside = Path.Combine(versions, StrayFolder);
Directory.CreateDirectory(aside);
File.Move(stray.Path, Path.Combine(aside, Path.GetFileName(stray.Path) + "." + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff", CultureInfo.InvariantCulture)));
Log.Warning("projects", $"Version file {stray.Path} does not follow the head of project {record.Id}; moved aside");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {stray.Path} could not be moved aside", ex);
}
foreach (var stray in files.Where(f => f.Number > (head?.Number ?? 0)))
SetAside(versions, stray.Path, $"does not follow the head of project {record.Id}");
}
if (!moved) return record;
var rolled = record with { Head = head, Updated = DateTimeOffset.UtcNow };
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(record.Id), rolled);
if (!writable) return rolled;
try
{
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(record.Id), rolled);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// The index takes the versions all the same; the record is written with them before anything else is (ProjectService).
Log.Warning("projects", $"The record of project {record.Id} could not be moved forward to version {head!.Number}", ex);
}
return rolled;
}
/// <summary>Writes what the index lacks of one project's record and versions.</summary>
private async Task IndexAsync(ProjectRecord record, string directory, CancellationToken cancellationToken)
/// <summary>A version file moved aside into <c>versions/.stray</c> (kept, never deleted).</summary>
private static void SetAside(string versions, string path, string why)
{
try
{
var aside = Path.Combine(versions, StrayFolder);
Directory.CreateDirectory(aside);
File.Move(path, Path.Combine(aside, Path.GetFileName(path) + "." + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff", CultureInfo.InvariantCulture)));
Log.Warning("projects", $"Version file {path} {why}; moved aside");
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {path} could not be moved aside", ex);
}
}
/// <summary>
/// Writes what the index lacks of one project's record and versions. False when its head could not be reached: the
/// record names a version this Chronos cannot read (damaged, of a newer Chronos), and the index's head stays on the
/// newest one it holds below it.
/// </summary>
private async Task<bool> IndexAsync(ProjectRecord record, string directory, bool trashed, bool writable, CancellationToken cancellationToken)
{
var row = await db.Projects.GetAsync(record.Id, cancellationToken).ConfigureAwait(false);
if (row is null)
{
var project = record.ToProject();
var live = await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
if (project.DeletedAt is null && project.IsMain && live.Any(p => p.IsMain))
if (project.DeletedAt is null)
{
Log.Warning("projects", $"Project {record.Id} says it is Main, but sequence {record.SequenceId} has one; indexed as another project");
project = project with { IsMain = false };
var live = await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
if (project.IsMain && live.Any(p => p.IsMain))
{
Log.Warning("projects", $"Project {record.Id} says it is Main, but sequence {record.SequenceId} has one; indexed as another project");
project = project with { IsMain = false };
}
project = project with { Name = FreeName(live, project.Name, project.Id) };
}
if (project.DeletedAt is null && live.Any(p => string.Equals(p.Name, project.Name, StringComparison.OrdinalIgnoreCase)))
project = project with { Name = project.Name + " (" + project.Id.ToString("N")[..6] + ")" };
// The head follows its versions below, so a version the index cannot take never leaves a head without its row.
await db.Projects.InsertAsync(project with { HeadVersionId = null, HeadNumber = 0 }, cancellationToken).ConfigureAwait(false);
row = project with { HeadVersionId = null, HeadNumber = 0 };
await db.Projects.InsertAsync(row, cancellationToken).ConfigureAwait(false);
}
var indexed = await db.Projects.ListVersionIdsAsync(record.Id, cancellationToken).ConfigureAwait(false);
var headNumber = row.HeadNumber;
Guid? headId = row.HeadVersionId;
var limit = record.Head?.Number ?? 0;
foreach (var file in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder)))
await IndexVersionsAsync(record, directory, writable, cancellationToken).ConfigureAwait(false);
var (headId, headNumber) = (row.HeadVersionId, row.HeadNumber);
var reached = true;
if (record.Head is { } head && (head.Number > headNumber || (head.Number == headNumber && head.Id != headId)))
{
if (file.Number > limit || indexed.ContainsKey(file.Number)) continue;
ProjectVersion version;
if (await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.ProjectId == record.Id
&& indexed.Number == head.Number)
{
(headId, headNumber) = (head.Id, head.Number);
}
else
{
reached = false;
Log.Warning("projects", $"The newest version of project {record.Id} ({head.Number}) cannot be read; the project is shown as far as it can be, and not changed");
var below = await db.Projects.ListVersionsAsync(record.Id, head.Number, 1, cancellationToken).ConfigureAwait(false);
if (below.Count > 0 && below[0].Number > headNumber) (headId, headNumber) = (below[0].Id, below[0].Number);
}
}
var updated = row with { HeadVersionId = headId, HeadNumber = headNumber };
if (record.Deleted is not null || trashed)
{
updated = updated with { DeletedAt = row.DeletedAt ?? record.Deleted };
}
else if (row.DeletedAt is not null || row.Name != record.Name)
{
var others = (await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
.Where(p => p.Id != record.Id).ToList();
if (row.DeletedAt is not null)
{
// Its live record says it is not deleted: listed again, Main again unless the sequence has another.
Log.Warning("projects", $"Project {record.Id} of sequence {record.SequenceId} is live in its sidecars; it is listed again");
updated = updated with { DeletedAt = null, IsMain = row.IsMain && !others.Any(p => p.IsMain) };
}
// The record's name, else (another project has it) the name the index has, else one free.
updated = updated with
{
Name = FreeName(others, record.Name, record.Id) == record.Name ? record.Name
: FreeName(others, row.Name, record.Id) == row.Name ? row.Name
: FreeName(others, record.Name, record.Id),
};
}
if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false);
return reached || record.Deleted is not null || trashed;
}
/// <summary>
/// The version files up to the record's head the index lacks, taken in. Of several files of one number, the one the
/// record's chain names is taken and the others are moved aside; when the chain cannot tell (a version on it cannot be
/// read), none is taken.
/// </summary>
private async Task IndexVersionsAsync(ProjectRecord record, string directory, bool writable, CancellationToken cancellationToken)
{
var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
var files = ProjectSidecars.ListVersions(versions);
var limit = record.Head?.Number ?? 0;
var indexed = new Dictionary<int, Guid>(await db.Projects.ListVersionIdsAsync(record.Id, cancellationToken).ConfigureAwait(false));
Dictionary<int, Guid>? chain = null;
foreach (var number in files.Where(f => f.Number <= limit && !indexed.ContainsKey(f.Number)).GroupBy(f => f.Number))
{
var candidates = number.ToList();
if (candidates.Count > 1)
{
chain ??= Chain(record, files);
var named = candidates.Where(c => chain.TryGetValue(c.Number, out var id) && id == c.Id).ToList();
if (named.Count == 0)
{
Log.Warning("projects", $"Project {record.Id} has {candidates.Count} version files numbered {number.Key} and its record's chain names none; none is indexed");
continue;
}
if (writable)
{
foreach (var other in candidates.Where(c => c.Id != named[0].Id))
SetAside(versions, other.Path, $"has the number of another version of project {record.Id}");
}
candidates = named;
}
foreach (var file in candidates)
{
ProjectVersion version;
try
{
version = ProjectSidecars.ReadVersion(file.Path);
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {file.Path} cannot be read; it is not indexed", ex);
continue;
}
if (version.ProjectId != record.Id || version.Number != file.Number || version.Id != file.Id) continue;
if (await db.Projects.GetVersionAsync(version.Id, cancellationToken).ConfigureAwait(false) is not null)
{
Log.Warning("projects", $"Version file {file.Path} holds a version the index has under another number; it is not indexed");
continue;
}
await db.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
indexed[version.Number] = version.Id;
}
}
}
/// <summary>The version ids by number along the record's chain, from its head back through each parent, as far as their files read.</summary>
private static Dictionary<int, Guid> Chain(ProjectRecord record, IReadOnlyList<(int Number, Guid Id, string Path)> files)
{
var chain = new Dictionary<int, Guid>();
var byId = files.GroupBy(f => f.Id).ToDictionary(g => g.Key, g => g.First());
for (var id = record.Head?.Id; id is { } current && byId.TryGetValue(current, out var file) && !chain.ContainsKey(file.Number);)
{
chain[file.Number] = file.Id;
try
{
version = ProjectSidecars.ReadVersion(file.Path);
id = ProjectSidecars.ReadVersion(file.Path).ParentId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {file.Path} cannot be read; it is not indexed", ex);
continue;
Log.Warning("projects", $"Version file {file.Path} cannot be read", ex);
break;
}
if (version.ProjectId != record.Id || version.Number != file.Number) continue;
await db.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
}
if (record.Head is { } head && head.Number > headNumber && await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is not null)
(headId, headNumber) = (head.Id, head.Number);
var updated = row with
{
Name = row.DeletedAt is null && record.Deleted is null && row.Name != record.Name && !await NameTakenAsync(record, cancellationToken).ConfigureAwait(false)
? record.Name
: row.Name,
HeadVersionId = headId,
HeadNumber = headNumber,
DeletedAt = row.DeletedAt ?? record.Deleted,
};
if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false);
return chain;
}
private async Task<bool> NameTakenAsync(ProjectRecord record, CancellationToken cancellationToken) =>
(await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
.Any(p => p.Id != record.Id && string.Equals(p.Name, record.Name, StringComparison.OrdinalIgnoreCase));
}
+431 -57
View File
@@ -35,6 +35,13 @@ public sealed class ProjectException(ProjectError reason, string message) : Inva
public ProjectError Reason { get; } = reason;
}
/// <summary>
/// What an operation needs of a sequence's projects cannot be read now: a record damaged, of a newer Chronos or not
/// readable for a moment, a head whose version this Chronos cannot read, a record ahead of the index. Nothing was written;
/// reads answer from the index meanwhile. Try again later (or with a newer Chronos).
/// </summary>
public sealed class ProjectsUnreadableException(string message) : IOException(message);
/// <summary>What became of a version pushed to its home (<see cref="ProjectService.PushAsync"/>).</summary>
public enum PushOutcome
{
@@ -53,8 +60,14 @@ public enum PushOutcome
/// <summary>The settings are of a newer format than this home reads: nothing was stored.</summary>
NewerFormat,
/// <summary>Not a version of this project: settings that are not portable (they name frames by position), or an id another project holds.</summary>
/// <summary>
/// Not a version of this project: settings that are not portable (they name frames by position) or that this home cannot
/// read, or an id another project holds.
/// </summary>
Invalid,
/// <summary>The version's file would be larger than a version may be (<see cref="ProjectSidecars.MaxVersionBytes"/>): nothing was stored.</summary>
TooLarge,
}
/// <summary>The answer to a push: the outcome, the version as stored (<see cref="PushOutcome.Stored"/>, <see cref="PushOutcome.AlreadyStored"/>) and the project's head.</summary>
@@ -76,6 +89,14 @@ public sealed record HomeFrames(int FramesVersion, IReadOnlyList<Frame> Frames,
/// <c>project.json</c> and settings (positions placed on its frames, through the settings version check), so the code
/// paths that read the sequence's settings keep working and only Main drives the hero frame the Library shows. Other
/// projects stay on this home: copies of a sequence linked to a server sync only its settings, i.e. Main.
/// <para>
/// The sidecars are the truth. A project the index holds is never taken for gone because its folder is missing or its
/// record cannot be read: a record that cannot be read holds its project back (listed as the index has it, nothing written
/// for it, no second Main made beside it), and a project whose folder is missing has its sidecars written again from the
/// index. A head of a newer settings format is listed and pulled (a workspace opens it read-only) but never mirrored nor
/// built on here. A read-only archive is read without writing to it: its projects are listed and pulled from its sidecars,
/// and a sequence with no Main yet gets one in the index only, whose sidecars are written once the archive takes writes.
/// </para>
/// </summary>
public sealed class ProjectService
{
@@ -87,6 +108,12 @@ public sealed class ProjectService
/// <summary>Frames read at once when a sequence's photos are gathered.</summary>
private const int PageSize = 1000;
/// <summary>Versions read at once when a project's sidecars are written again from the index.</summary>
private const int WriteBackPage = 500;
/// <summary>How far back from Main's head an index that knows no mirror looks for the version its sequence's settings hold.</summary>
private const int MirrorSearch = 100;
/// <summary>How long a read waits for a sequence another writer holds before it answers from the index alone.</summary>
private static readonly TimeSpan ReadWait = TimeSpan.FromSeconds(3);
@@ -104,6 +131,9 @@ public sealed class ProjectService
/// <summary>The client name versions made here carry (the archive engine's lease owner).</summary>
public string ClientName => _writer.Options.Owner;
/// <summary>Whether an archive takes writes now (<see cref="ArchiveVolume.Probe"/>; tests stand in a read-only one).</summary>
internal Func<ArchivePaths, ArchiveAvailability> Availability { get; set; } = ArchiveVolume.Probe;
/// <summary>The live projects of a sequence, Main first; read from the index alone while the archive is offline.</summary>
/// <exception cref="ProjectException">The sequence is not in the index.</exception>
public async Task<IReadOnlyList<SequenceProject>> ListAsync(Guid sequenceId, CancellationToken cancellationToken = default)
@@ -112,16 +142,18 @@ public sealed class ProjectService
return await _db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
}
/// <summary>The sequence's Main project, made now (from its settings) when it has none.</summary>
/// <summary>The sequence's Main project, made now (from its settings) when it has none; in the index only while the archive is read-only.</summary>
/// <exception cref="ProjectException">The sequence is not in the index.</exception>
/// <exception cref="ArchiveOfflineException">The archive is offline and the index has no Main project yet.</exception>
/// <exception cref="ProjectsUnreadableException">The index has no Main and a project record cannot be read now (it may be Main's).</exception>
public async Task<SequenceProject> MainAsync(Guid sequenceId, CancellationToken cancellationToken = default)
{
await ReadAsync(sequenceId, cancellationToken).ConfigureAwait(false);
if (await _db.Projects.GetMainAsync(sequenceId, cancellationToken).ConfigureAwait(false) is { } main) return main;
// The catch-up could not run (the archive offline, the sequence busy): it runs now, or says why it cannot.
return await UnderLeaseAsync(sequenceId, async (_, ct) => (await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false))!, cancellationToken)
.ConfigureAwait(false);
return await UnderLeaseAsync(sequenceId, async (_, ct) => await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false)
?? throw new ProjectsUnreadableException($"The Main of sequence {sequenceId} cannot be read now."),
cancellationToken).ConfigureAwait(false);
}
public Task<SequenceProject?> GetAsync(Guid projectId, CancellationToken cancellationToken = default) => _db.Projects.GetAsync(projectId, cancellationToken);
@@ -149,7 +181,8 @@ public sealed class ProjectService
else
{
var main = (await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false))!;
source = (await _db.Projects.GetVersionAsync(main.HeadVersionId!.Value, ct).ConfigureAwait(false))!;
source = await _db.Projects.GetVersionAsync(main.HeadVersionId ?? Guid.Empty, ct).ConfigureAwait(false)
?? throw new ProjectsUnreadableException($"Main of sequence {sequenceId} has no version this Chronos can read.");
}
var now = DateTimeOffset.UtcNow;
var project = new SequenceProject
@@ -176,7 +209,7 @@ public sealed class ProjectService
CreatedAt = now,
StoredAt = now,
};
return await AddProjectAsync(context.Folder, project, first, ct).ConfigureAwait(false);
return await AddProjectAsync(context.Folder, project, first, null, ct).ConfigureAwait(false);
}, cancellationToken);
}
@@ -191,16 +224,18 @@ public sealed class ProjectService
var current = await LiveAsync(projectId, ct).ConfigureAwait(false);
if (current.Name == trimmed) return current;
await EnsureNameFreeAsync(current.SequenceId, projectId, trimmed, ct).ConfigureAwait(false);
var record = OwnRecord(context, projectId);
var renamed = current with { Name = trimmed, UpdatedAt = DateTimeOffset.UtcNow };
ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(renamed));
ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(renamed, record.Mirrored));
await _db.Projects.UpdateAsync(renamed, ct).ConfigureAwait(false);
return renamed;
}, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Deletes a project: its record says so, its folder moves to <c>.chronos/projects/.trash</c> (kept, never erased by
/// Chronos) and it leaves the listing.
/// Deletes a project: its folder moves to <c>.chronos/projects/.trash</c> (kept, never erased by Chronos), then its
/// record there says so, and it leaves the listing. A folder that cannot be moved (a file in it held open) fails the
/// delete with nothing changed.
/// </summary>
/// <exception cref="ProjectException">Unknown project, or the Main project.</exception>
public async Task DeleteAsync(Guid projectId, CancellationToken cancellationToken = default)
@@ -210,12 +245,22 @@ public sealed class ProjectService
await UnderLeaseAsync(project.SequenceId, async (context, ct) =>
{
var current = await LiveAsync(projectId, ct).ConfigureAwait(false);
var record = OwnRecord(context, projectId);
var now = DateTimeOffset.UtcNow;
var deleted = current with { DeletedAt = now, UpdatedAt = now };
ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(deleted));
Directory.CreateDirectory(context.Folder.ProjectsTrashDirectory);
Directory.Move(context.Folder.ProjectDirectory(projectId),
Path.Combine(context.Folder.ProjectsTrashDirectory, projectId.ToString("N") + "-" + now.UtcTicks.ToString(System.Globalization.CultureInfo.InvariantCulture)));
var trashed = Path.Combine(context.Folder.ProjectsTrashDirectory,
projectId.ToString("N") + "-" + now.UtcTicks.ToString(System.Globalization.CultureInfo.InvariantCulture));
Directory.Move(context.Folder.ProjectDirectory(projectId), trashed);
try
{
ProjectSidecars.WriteRecord(Path.Combine(trashed, ArchiveLayout.ProjectFile), ProjectRecord.Of(deleted, record.Mirrored));
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// In the projects' trash it is deleted all the same (ProjectIndexer): the record only says when.
Log.Warning("projects", $"The record of deleted project {projectId} could not say so", ex);
}
await _db.Projects.UpdateAsync(deleted, ct).ConfigureAwait(false);
return true;
}, cancellationToken).ConfigureAwait(false);
@@ -275,10 +320,13 @@ public sealed class ProjectService
? new PushResult(PushOutcome.AlreadyStored, existing, head)
: new PushResult(PushOutcome.Invalid, null, head);
}
// A project whose record or head cannot be read now takes nothing: its history may go on past what the index holds.
if (context.Reconcile.HeldBack.Contains(projectId))
throw new ProjectsUnreadableException($"Project {projectId} cannot be read now; the version is kept to be sent again.");
if (!ProjectContent.Verifies(version.SettingsJson, version.ContentHash)) return new PushResult(PushOutcome.HashMismatch, null, head);
var format = ProjectContent.FormatOf(version.SettingsJson);
if (format > ProjectSettings.CurrentVersion) return new PushResult(PushOutcome.NewerFormat, null, head);
if (!IsPortable(version.SettingsJson)) return new PushResult(PushOutcome.Invalid, null, head);
if (!IsPortable(version.SettingsJson) || !Readable(version.SettingsJson)) return new PushResult(PushOutcome.Invalid, null, head);
if (version.ParentId != current.HeadVersionId) return new PushResult(PushOutcome.Diverged, null, head);
var stored = version with
{
@@ -288,8 +336,21 @@ public sealed class ProjectService
SettingsFormat = format,
StoredAt = DateTimeOffset.UtcNow,
};
// Never stored when it could not be read back (the same limit for both).
if (!ProjectSidecars.Fits(stored)) return new PushResult(PushOutcome.TooLarge, null, head);
current = await AppendAsync(context.Folder, current, stored, ct).ConfigureAwait(false);
if (current.IsMain) await MirrorAsync(context, current, stored, ct).ConfigureAwait(false);
if (current.IsMain)
{
try
{
await MirrorAsync(context, current, stored, ct).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// The version is stored: the next operation's catch-up completes the mirror.
Log.Warning("projects", $"Version {stored.Number} of Main of sequence {current.SequenceId} is stored but not mirrored yet", ex);
}
}
return new PushResult(PushOutcome.Stored, stored, stored);
}, cancellationToken).ConfigureAwait(false);
}
@@ -307,9 +368,42 @@ public sealed class ProjectService
}
}
/// <summary>
/// Whether this archive holds <paramref name="version"/> of the project durably: its file, read back now, is that version
/// with that content hash (the index's row alone proves nothing: a file damaged or taken away since is not seen there).
/// False when the file is missing, damaged, of a newer Chronos or another version's, or the project was deleted: what a
/// workspace checks before it drops its own copy of a version.
/// </summary>
/// <exception cref="ArchiveOfflineException">The archive is offline.</exception>
/// <exception cref="SequenceFolderMovedException">The sequence's folder does not hold it now (another index of the archive moved it).</exception>
public async Task<bool> StoredAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(version);
if (await _db.Projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) is not { DeletedAt: null } project) return false;
var (paths, row) = await LocateAsync(project.SequenceId, cancellationToken).ConfigureAwait(false);
if (Availability(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, project.SequenceId)) throw new SequenceFolderMovedException();
var number = (await _db.Projects.GetVersionAsync(version.Id, cancellationToken).ConfigureAwait(false))?.Number ?? version.Number;
var path = folder.VersionPath(projectId, number, version.Id);
try
{
var stored = ProjectSidecars.ReadVersion(path);
if (stored.Id == version.Id && stored.ProjectId == projectId && string.Equals(stored.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase))
return true;
Log.Warning("projects", $"{path} does not hold version {version.Id} as it was sent");
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version {number} of project {projectId} cannot be read back from the archive", ex);
}
return false;
}
/// <summary>
/// Catches the sequence's projects up under its lease (see the class remarks) before a read; the index alone answers while
/// the archive is offline or the sequence busy for longer than the lease wait.
/// the archive is offline, the sequence busy for longer than the lease wait, or a project of it cannot be read now. A
/// read-only archive is caught up without writing to it (<see cref="ReadOnlyCatchUpAsync"/>).
/// </summary>
private async Task ReadAsync(Guid sequenceId, CancellationToken cancellationToken)
{
@@ -317,7 +411,38 @@ public sealed class ProjectService
{
await UnderLeaseAsync(sequenceId, (_, _) => Task.FromResult(true), cancellationToken, ReadWait).ConfigureAwait(false);
}
catch (Exception ex) when (ex is ArchiveOfflineException or LeaseUnavailableException or SequenceFolderMovedException)
catch (ArchiveOfflineException ex) when (ex.Availability == ArchiveAvailability.ReadOnly)
{
await ReadOnlyCatchUpAsync(sequenceId, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is ArchiveOfflineException or LeaseUnavailableException or SequenceFolderMovedException or ProjectsUnreadableException)
{
Log.Info("projects", $"The projects of sequence {sequenceId} are read from the index: {ex.Message}");
}
}
/// <summary>
/// The catch-up of a sequence of a read-only archive (a backup mounted read-only, a share this PC may only read): its
/// sidecars read into the index as they are and nothing written to the archive (no lease, no record moved forward, no
/// version set aside, no mirror). A sequence with no Main anywhere gets one in the index only, from its settings, so it
/// opens; its sidecars are written once the archive takes writes (<see cref="WriteBackAsync"/>).
/// </summary>
private async Task ReadOnlyCatchUpAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var (paths, row) = await LocateAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var folder = paths.Sequence(row.Folder);
try
{
if (!SequenceFolders.Holds(folder, sequenceId))
{
Log.Info("projects", $"The folder of sequence {sequenceId} does not hold it now; its projects are read from the index");
return;
}
var context = new LeaseContext(folder, row, await _indexer.ReconcileAsync(folder, sequenceId, cancellationToken, writable: false).ConfigureAwait(false));
if (context.Reconcile.Complete && !await HasMainAsync(context, cancellationToken).ConfigureAwait(false))
await MakeMainAsync(context, writable: false, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Info("projects", $"The projects of sequence {sequenceId} are read from the index: {ex.Message}");
}
@@ -334,34 +459,66 @@ public sealed class ProjectService
private async Task<T> UnderLeaseAsync<T>(Guid sequenceId, Func<LeaseContext, CancellationToken, Task<T>> work, CancellationToken cancellationToken,
TimeSpan? wait = null)
{
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
var paths = new ArchivePaths(archive.RootPath);
ArchiveVolume.EnsureWritable(paths);
var (paths, _) = await LocateAsync(sequenceId, cancellationToken).ConfigureAwait(false);
var availability = Availability(paths);
if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(paths.Root, availability);
await using (await _writer.LockAsync(paths, sequenceId, wait, cancellationToken).ConfigureAwait(false))
{
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, sequenceId)) throw new SequenceFolderMovedException();
var context = new LeaseContext(folder, row);
var context = new LeaseContext(folder, row, await _indexer.ReconcileAsync(folder, sequenceId, cancellationToken).ConfigureAwait(false));
await CatchUpAsync(context, cancellationToken).ConfigureAwait(false);
return await work(context, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>The sidecars into the index, the Main project made, and Main in step with the sequence's settings.</summary>
/// <summary>The archive of a sequence and its index row.</summary>
private async Task<(ArchivePaths Paths, Sequence Row)> LocateAsync(Guid sequenceId, CancellationToken cancellationToken)
{
var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
return (new ArchivePaths(archive.RootPath), sequence);
}
/// <summary>
/// After the sidecars came into the index (<see cref="ProjectIndexer"/>): the projects the archive has no folder for
/// written back, the Main project made when the sequence has none, and Main in step with the sequence's settings.
/// </summary>
private async Task CatchUpAsync(LeaseContext context, CancellationToken cancellationToken)
{
await _indexer.ReconcileAsync(context.Folder, context.Row.Id, cancellationToken).ConfigureAwait(false);
await WriteBackAsync(context, cancellationToken).ConfigureAwait(false);
var main = await _db.Projects.GetMainAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
if (main?.HeadVersionId is null)
if (main is null)
{
await MakeMainAsync(context, main, cancellationToken).ConfigureAwait(false);
// A record that cannot be read now may be Main's: no second Main is made beside it.
if (!context.Reconcile.Complete)
throw new ProjectsUnreadableException($"Sequence {context.Row.Id} has no Main in the index and a project of it cannot be read now.");
if (!await HasMainAsync(context, cancellationToken).ConfigureAwait(false))
await MakeMainAsync(context, writable: true, cancellationToken).ConfigureAwait(false);
return;
}
if (context.Reconcile.HeldBack.Contains(main.Id) || main.HeadVersionId is not { } headId)
{
// Nothing is mirrored from, merged into or built on a Main whose record or head cannot be read now.
Log.Info("projects", $"Main of sequence {context.Row.Id} cannot be read now; it is caught up once it can");
return;
}
if (main.MirroredVersionId == headId && main.MirroredSettingsVersion == context.Row.SettingsVersion) return;
var head = (await _db.Projects.GetVersionAsync(headId, cancellationToken).ConfigureAwait(false))!;
if (IsNewer(head))
{
// A head of a newer Chronos (another PC sharing the archive): listed and pulled (a workspace opens it read-only), never
// mirrored, merged into or built on here; the sequence's settings stay as that Chronos wrote them.
Log.Info("projects", $"Main of sequence {context.Row.Id} is at version {head.Number} of a newer Chronos; the sequence's settings are left to it");
return;
}
if (main.MirroredVersionId == main.HeadVersionId && main.MirroredSettingsVersion == context.Row.SettingsVersion) return;
var head = (await _db.Projects.GetVersionAsync(main.HeadVersionId.Value, cancellationToken).ConfigureAwait(false))!;
var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
if (main.MirroredVersionId is null)
{
await MeetMainAsync(context, main, head, frames.Photos, cancellationToken).ConfigureAwait(false);
return;
}
if (CurrentPortable(context, frames.Photos) is not { } current)
{
// Settings no Chronos can read (or of a newer one): left as they are, recorded as seen.
@@ -370,10 +527,10 @@ public sealed class ProjectService
}
// The head parks its photos as the frames held them when it was made: a conversion or a hash since names them anew.
var sameAsHead = ProjectContent.SameChoices(head.SettingsJson, current, frames.Photos);
if (main.MirroredVersionId == main.HeadVersionId || main.MirroredVersionId is null)
if (main.MirroredVersionId == main.HeadVersionId)
{
// Only the sequence's settings version moved (settings another path wrote, or only its title or tags), or the index
// was rebuilt and knows no mirror: settings that differ from the head are its newest state.
// Only the sequence's settings version moved (settings another path wrote, or only its title or tags): settings that
// differ from the head are its newest state.
var latest = sameAsHead ? head : await AppendDerivedAsync(context, main, head, current, ProjectVersionKind.External, cancellationToken).ConfigureAwait(false);
await _db.Projects.SetMirroredAsync(main.Id, latest.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
return;
@@ -401,13 +558,145 @@ public sealed class ProjectService
await MirrorAsync(context, main, version, cancellationToken).ConfigureAwait(false);
}
/// <summary>The Main project of a sequence that has none (or one without versions): version 1 from the sequence's settings.</summary>
private async Task MakeMainAsync(LeaseContext context, SequenceProject? existing, CancellationToken cancellationToken)
/// <summary>
/// Main as this index meets it with no mirror known (an index rebuilt from the sidecars, another PC's index of the archive,
/// an index taken back from a backup): the sequence's settings are what its <c>project.json</c> holds, the file every
/// mirror writes, never this index's copy, which may have been read long ago. The head's choices: in step, this index's
/// copy brought up to them. Those of the version Main's record says was mirrored last (an older record names none: of a
/// version the head descends from): a mirror a crash cut short, completed. Neither: settings another path wrote since,
/// an external version.
/// </summary>
private async Task MeetMainAsync(LeaseContext context, SequenceProject main, ProjectVersion head, FramePhotos photos, CancellationToken cancellationToken)
{
if (SharedPortable(context, photos) is not { } shared)
{
// Settings no Chronos can read (or of a newer one): left as they are, recorded as seen.
await _db.Projects.SetMirroredAsync(main.Id, head.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
return;
}
if (ProjectContent.SameChoices(head.SettingsJson, shared, photos))
{
if (CurrentPortable(context, photos) is { } copy && ProjectContent.SameChoices(head.SettingsJson, copy, photos))
await _db.Projects.SetMirroredAsync(main.Id, head.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
else
await MirrorAsync(context, main, head, cancellationToken).ConfigureAwait(false);
return;
}
if (await MirroredBeforeAsync(context, main, head, shared, photos, cancellationToken).ConfigureAwait(false))
{
await MirrorAsync(context, main, head, cancellationToken).ConfigureAwait(false);
return;
}
var external = await AppendDerivedAsync(context, main, head, shared, ProjectVersionKind.External, cancellationToken).ConfigureAwait(false);
main = (await _db.Projects.GetAsync(main.Id, cancellationToken).ConfigureAwait(false))!;
await MirrorAsync(context, main, external, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Whether <paramref name="shared"/> holds the choices of an earlier version of Main a mirror wrote: the one Main's record
/// names as mirrored last, else (an older record) one of the versions the head descends from.
/// </summary>
private async Task<bool> MirroredBeforeAsync(LeaseContext context, SequenceProject main, ProjectVersion head, string shared, FramePhotos photos,
CancellationToken cancellationToken)
{
if (context.Reconcile.Live.TryGetValue(main.Id, out var record) && record.Mirrored is { } recorded)
{
return recorded.Id != head.Id && await _db.Projects.GetVersionAsync(recorded.Id, cancellationToken).ConfigureAwait(false) is { } mirrored
&& mirrored.ProjectId == main.Id && !IsNewer(mirrored) && ProjectContent.SameChoices(mirrored.SettingsJson, shared, photos);
}
var parent = head.ParentId;
for (var step = 0; parent is { } id && step < MirrorSearch; step++)
{
if (await _db.Projects.GetVersionAsync(id, cancellationToken).ConfigureAwait(false) is not { } version || version.ProjectId != main.Id) break;
if (!IsNewer(version) && ProjectContent.SameChoices(version.SettingsJson, shared, photos)) return true;
parent = version.ParentId;
}
return false;
}
/// <summary>
/// Whether the sequence has a Main the archive does not say is gone: a live row, or one this index marked deleted (an
/// older Chronos took a record it could not read for gone) whose folder the archive does not have.
/// </summary>
private async Task<bool> HasMainAsync(LeaseContext context, CancellationToken cancellationToken) =>
(await _db.Projects.ListBySequenceAsync(context.Row.Id, includeDeleted: true, cancellationToken).ConfigureAwait(false))
.Any(p => p.IsMain && (p.DeletedAt is null || !context.Reconcile.Seen.Contains(p.Id)));
/// <summary>
/// The projects the index holds that the archive has no folder for (a projects folder moved away and put back without
/// them, a sync tool, a Main made while the archive was read-only), with a Main this index marked deleted only because an
/// older Chronos took its unreadable record for gone: their sidecars written again from the index, which holds every
/// version, so the archive is their truth again and nothing is lost. Never over a folder that is there, nor into a
/// sequence folder without its sidecar. A Main beside which the archive has another live Main record is written back as
/// another project (the one in the archive stays Main). A project that cannot be written is tried again next time.
/// </summary>
private async Task WriteBackAsync(LeaseContext context, CancellationToken cancellationToken)
{
if (!File.Exists(context.Folder.ManifestPath)) return;
var rows = await _db.Projects.ListBySequenceAsync(context.Row.Id, includeDeleted: true, cancellationToken).ConfigureAwait(false);
var liveMain = rows.Any(p => p.IsMain && p.DeletedAt is null);
foreach (var row in rows.Where(p => !context.Reconcile.Seen.Contains(p.Id) && (p.DeletedAt is null || (p.IsMain && !liveMain))))
{
if (Directory.Exists(context.Folder.ProjectDirectory(row.Id))) continue;
var others = (await _db.Projects.ListBySequenceAsync(context.Row.Id, includeDeleted: false, cancellationToken).ConfigureAwait(false))
.Where(p => p.Id != row.Id).ToList();
var project = row with { DeletedAt = null, Name = ProjectIndexer.FreeName(others, row.Name, row.Id) };
SequenceProject? promoted = null;
if (project.IsMain && context.Reconcile.Live.Values.FirstOrDefault(r => r.IsMain && r.Id != row.Id) is { } theirs
&& others.FirstOrDefault(p => p.Id == theirs.Id) is { } other)
{
// Another index made the sequence's Main meanwhile, into the archive: that one is Main, this one another project.
var rest = others.Where(p => p.Id != other.Id).ToList();
promoted = other with { IsMain = true, Name = ProjectIndexer.FreeName(rest, theirs.Name, other.Id) };
project = project with
{
IsMain = false,
Name = ProjectIndexer.FreeName([.. rest, promoted], row.Name, row.Id),
MirroredVersionId = null,
MirroredSettingsVersion = null,
};
}
try
{
for (var after = 0; ;)
{
var page = await _db.Projects.ListVersionsAfterAsync(row.Id, after, WriteBackPage, cancellationToken).ConfigureAwait(false);
foreach (var version in page) ProjectSidecars.WriteVersion(context.Folder.VersionPath(row.Id, version.Number, version.Id), version);
if (page.Count < WriteBackPage) break;
after = page[^1].Number;
}
var mirrored = project.IsMain && project.MirroredVersionId is { } mirroredId
&& await _db.Projects.GetVersionAsync(mirroredId, cancellationToken).ConfigureAwait(false) is { } known
? new ProjectHead(known.Id, known.Number)
: null;
ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(row.Id), ProjectRecord.Of(project, mirrored));
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"The sidecars of project {row.Id} could not be written again from the index; tried again next time", ex);
continue;
}
Log.Warning("projects", $"Project {row.Id} of sequence {context.Row.Id} had no folder in the archive; its sidecars are written again from the index");
if (project == row) continue;
await _db.Projects.UpdateAsync(project, cancellationToken).ConfigureAwait(false);
if (promoted is not null) await _db.Projects.UpdateAsync(promoted, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// The Main project of a sequence that has none: version 1 from the sequence's settings as its <c>project.json</c> holds
/// them (this index's copy when there is none, else defaults), this index's copy brought up to them. With the archive
/// read-only it is made in the index only, and nothing is written to the archive: its sidecars are written once the
/// archive takes writes (<see cref="WriteBackAsync"/>).
/// </summary>
private async Task MakeMainAsync(LeaseContext context, bool writable, CancellationToken cancellationToken)
{
var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
var portable = CurrentPortable(context, frames.Photos) ?? ProjectContent.ToPortable(new ProjectSettings(), frames.Photos);
var shared = SharedPortable(context, frames.Photos);
var copy = CurrentPortable(context, frames.Photos);
var portable = shared ?? copy ?? ProjectContent.ToPortable(new ProjectSettings(), frames.Photos);
var now = DateTimeOffset.UtcNow;
var project = existing ?? new SequenceProject
var project = new SequenceProject
{
Id = Guid.CreateVersion7(),
SequenceId = context.Row.Id,
@@ -416,17 +705,19 @@ public sealed class ProjectService
CreatedAt = now,
UpdatedAt = now,
};
var first = NewVersion(project.Id, null, project.HeadNumber + 1, ProjectVersionKind.Imported, portable, context.Row.FramesVersion, now);
if (existing is null)
var first = NewVersion(project.Id, null, 1, ProjectVersionKind.Imported, portable, context.Row.FramesVersion, now);
// Settings this Chronos cannot read (of a newer one) are never written over: the index's copy then stays as it is too.
var inStep = shared is null || copy is null || ProjectContent.SameChoices(portable, copy, frames.Photos);
if (inStep) project = project with { MirroredSettingsVersion = context.Row.SettingsVersion, MirroredVersionId = first.Id };
if (!writable)
{
await AddProjectAsync(context.Folder, project with { MirroredSettingsVersion = context.Row.SettingsVersion, MirroredVersionId = first.Id }, first,
cancellationToken).ConfigureAwait(false);
}
else
{
var updated = await AppendAsync(context.Folder, existing, first, cancellationToken).ConfigureAwait(false);
await _db.Projects.SetMirroredAsync(updated.Id, first.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
await AddProjectAsync(null, project, first, null, cancellationToken).ConfigureAwait(false);
return;
}
var onDisk = shared is not null && File.Exists(context.Folder.ProjectPath);
var made = await AddProjectAsync(context.Folder, project, first, onDisk ? new ProjectHead(first.Id, first.Number) : null, cancellationToken)
.ConfigureAwait(false);
if (!inStep) await MirrorAsync(context, made, first, cancellationToken).ConfigureAwait(false);
}
/// <summary>"Main", or "Main (2)"… when a project of that name exists already (one made before the sequence had its Main).</summary>
@@ -440,10 +731,19 @@ public sealed class ProjectService
}
/// <summary>The portable form of the sequence's settings in force (index, else <c>project.json</c>, else defaults); null when unreadable or newer.</summary>
private static string? CurrentPortable(LeaseContext context, FramePhotos photos)
private static string? CurrentPortable(LeaseContext context, FramePhotos photos) =>
Portable(context, context.Row.SettingsJson ?? (File.Exists(context.Folder.ProjectPath) ? AtomicFile.ReadAllText(context.Folder.ProjectPath) : null), photos);
/// <summary>
/// The portable form of the sequence's settings as the archive holds them, in <c>project.json</c> (else this index's copy,
/// else defaults); null when unreadable or newer.
/// </summary>
private static string? SharedPortable(LeaseContext context, FramePhotos photos) =>
Portable(context, File.Exists(context.Folder.ProjectPath) ? AtomicFile.ReadAllText(context.Folder.ProjectPath) : context.Row.SettingsJson, photos);
private static string? Portable(LeaseContext context, string? json, FramePhotos photos)
{
var json = context.Row.SettingsJson ?? (File.Exists(context.Folder.ProjectPath) ? AtomicFile.ReadAllText(context.Folder.ProjectPath) : null)
?? new ProjectSettings().ToJson();
json ??= new ProjectSettings().ToJson();
try
{
if (ProjectContent.FormatOf(json) > ProjectSettings.CurrentVersion) return null;
@@ -456,6 +756,10 @@ public sealed class ProjectService
}
}
/// <summary>Whether a version holds settings of a newer Chronos (shown and pulled, never mirrored nor built on here).</summary>
private static bool IsNewer(ProjectVersion version) =>
version.SettingsFormat > ProjectSettings.CurrentVersion || ProjectContent.FormatOf(version.SettingsJson) > ProjectSettings.CurrentVersion;
/// <summary>A version made here (catch-up or merge) appended to Main.</summary>
private async Task<ProjectVersion> AppendDerivedAsync(LeaseContext context, SequenceProject main, ProjectVersion head, string portable, ProjectVersionKind kind,
CancellationToken cancellationToken)
@@ -481,12 +785,19 @@ public sealed class ProjectService
StoredAt = now,
};
/// <summary>A new project with its first version: sidecars (version, then record), then both rows in one transaction.</summary>
private async Task<SequenceProject> AddProjectAsync(SequencePaths folder, SequenceProject project, ProjectVersion first, CancellationToken cancellationToken)
/// <summary>
/// A new project with its first version: sidecars (version, then the record, naming <paramref name="mirrored"/>), then
/// both rows in one transaction; the rows only when <paramref name="folder"/> is null (a read-only archive).
/// </summary>
private async Task<SequenceProject> AddProjectAsync(SequencePaths? folder, SequenceProject project, ProjectVersion first, ProjectHead? mirrored,
CancellationToken cancellationToken)
{
var withHead = project with { HeadVersionId = first.Id, HeadNumber = first.Number };
ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, first.Number, first.Id), first);
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(withHead));
if (folder is not null)
{
ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, first.Number, first.Id), first);
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(withHead, mirrored));
}
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
await tx.Projects.InsertAsync(withHead with { HeadVersionId = null, HeadNumber = 0 }, cancellationToken).ConfigureAwait(false);
await tx.Projects.InsertVersionAsync(first, cancellationToken).ConfigureAwait(false);
@@ -495,13 +806,21 @@ public sealed class ProjectService
return withHead;
}
/// <summary>A version appended to a project: its sidecar, the record's head, then the version row and the head in one transaction.</summary>
/// <summary>
/// A version appended to a project: its sidecar, the record's head, then the version row and the head in one
/// transaction. Only onto the head the record names, read again now: a record that cannot be read, or that names another
/// head than the index has (a version this Chronos cannot read), is never written over and the version is not made.
/// </summary>
/// <exception cref="ProjectsUnreadableException">The record cannot be read, or its head is not the index's.</exception>
private async Task<SequenceProject> AppendAsync(SequencePaths folder, SequenceProject project, ProjectVersion version, CancellationToken cancellationToken)
{
var record = ReadOwnRecord(folder, project.Id);
if (record.Head != (project.HeadVersionId is { } headId ? new ProjectHead(headId, project.HeadNumber) : null))
throw new ProjectsUnreadableException($"The record of project {project.Id} names version {record.Head?.Number} as its head, which the index does not hold.");
var now = DateTimeOffset.UtcNow;
var advanced = project with { HeadVersionId = version.Id, HeadNumber = version.Number, UpdatedAt = now };
ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, version.Number, version.Id), version);
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(advanced));
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(advanced, record.Mirrored));
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
await tx.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
if (!await tx.Projects.AdvanceHeadAsync(project.Id, project.HeadNumber, version.Id, version.Number, now, cancellationToken).ConfigureAwait(false))
@@ -510,18 +829,58 @@ public sealed class ProjectService
return advanced;
}
/// <summary>The record of a live project as its folder holds it now, before it is written again: one held back or unreadable is never written over.</summary>
/// <exception cref="ProjectsUnreadableException">The project is held back, or its record cannot be read.</exception>
private static ProjectRecord OwnRecord(LeaseContext context, Guid projectId) =>
context.Reconcile.HeldBack.Contains(projectId)
? throw new ProjectsUnreadableException($"Project {projectId} cannot be read now.")
: ReadOwnRecord(context.Folder, projectId);
private static ProjectRecord ReadOwnRecord(SequencePaths folder, Guid projectId)
{
try
{
return ProjectSidecars.ReadRecord(folder.ProjectRecordPath(projectId));
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
throw new ProjectsUnreadableException($"The record of project {projectId} cannot be read now: {ex.Message}");
}
}
/// <summary>
/// Writes Main's <paramref name="version"/> into the sequence: its positions placed on the sequence's frames, then
/// <c>project.json</c> and the index's settings (and hero frame) with the settings version as the check. Recorded as
/// mirrored when the index took it; a settings edit that won the race is caught up by the next operation (merged).
/// <c>project.json</c>, Main's record naming it as mirrored, and the index's settings (and hero frame) with the settings
/// version as the check. Recorded as mirrored when the index took it; a settings edit that won the race is caught up by
/// the next operation (merged). A version this Chronos cannot read (of a newer one) is never mirrored: nothing is written.
/// </summary>
private async Task MirrorAsync(LeaseContext context, SequenceProject main, ProjectVersion version, CancellationToken cancellationToken)
{
var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
var placed = ProjectSettings.FromJson(ProjectContent.Place(version.SettingsJson, frames.Photos));
ProjectSettings placed;
try
{
if (IsNewer(version)) throw new InvalidDataException($"Version {version.Number} of Main is of a newer Chronos.");
placed = ProjectSettings.FromJson(ProjectContent.Place(version.SettingsJson, frames.Photos));
}
catch (Exception ex) when (ex is JsonException or InvalidDataException)
{
Log.Warning("projects", $"Version {version.Number} of Main of sequence {context.Row.Id} cannot be mirrored into its settings; they stay as they are", ex);
return;
}
var json = placed.ToJson();
Directory.CreateDirectory(context.Folder.MetaDirectory);
AtomicFile.WriteAllText(context.Folder.ProjectPath, json);
try
{
var record = ProjectSidecars.ReadRecord(context.Folder.ProjectRecordPath(main.Id));
ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(main.Id), record with { Mirrored = new ProjectHead(version.Id, version.Number) });
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
// An index meeting Main then compares project.json with the versions the head descends from.
Log.Warning("projects", $"The record of Main of sequence {context.Row.Id} could not name version {version.Number} as mirrored", ex);
}
var hero = placed.HeroFrame is { } index && (uint)index < (uint)frames.Frames.Count ? frames.Frames[index].Id : Guid.Empty;
var result = await _db.Sequences.EditAsync(context.Row.Id, new SequenceEdit { SettingsJson = json, HeroFrameId = hero }, context.Row.SettingsVersion,
context.Row.FramesVersion, cancellationToken).ConfigureAwait(false);
@@ -536,6 +895,20 @@ public sealed class ProjectService
}
}
/// <summary>Whether settings this Chronos reads strictly (a member of the wrong type is refused): a version that can be mirrored and edited.</summary>
private static bool Readable(string json)
{
try
{
ProjectSettings.FromJson(json);
return true;
}
catch (Exception ex) when (ex is JsonException or InvalidDataException)
{
return false;
}
}
/// <summary>Whether settings name no frame by position (every position parked by photo).</summary>
private static bool IsPortable(string json)
{
@@ -592,10 +965,11 @@ public sealed class ProjectService
private static ProjectException SequenceMissing(Guid sequenceId) => new(ProjectError.SequenceNotFound, $"Sequence {sequenceId} is not in the index.");
/// <summary>The sequence as read under its lease (the mirror moves its settings version) and its folder.</summary>
private sealed class LeaseContext(SequencePaths folder, Sequence row)
/// <summary>The sequence as read under its lease (the mirror moves its settings version), its folder, and what its projects' reconcile met.</summary>
private sealed class LeaseContext(SequencePaths folder, Sequence row, ProjectReconcile reconcile)
{
public SequencePaths Folder { get; } = folder;
public Sequence Row { get; set; } = row;
public ProjectReconcile Reconcile { get; } = reconcile;
}
}
+44 -14
View File
@@ -11,20 +11,26 @@ namespace Chronos.Data.Projects;
/// <summary>The newest version of a project as its record names it: id and number.</summary>
public sealed record ProjectHead(Guid Id, int Number);
/// <summary>A version's file would be over <see cref="ProjectSidecars.MaxVersionBytes"/>, so it could never be read back: nothing was written.</summary>
public sealed class ProjectVersionTooLargeException(string message) : IOException(message);
/// <summary>
/// The record of a project in its folder (<c>.chronos/projects/&lt;id&gt;/project.json</c>): what the index row is rebuilt
/// from. <see cref="Deleted"/> is set when the project was deleted, before its folder moves to the projects' trash.
/// from. <see cref="Deleted"/> is set when the project was deleted, once its folder is in the projects' trash.
/// </summary>
/// <param name="Mirrored">Main only: the version whose settings a mirror last wrote into the sequence's <c>project.json</c>
/// (any index of the archive). An index that meets Main for the first time compares <c>project.json</c> with it, never
/// with a copy of the settings it may have read long ago (<see cref="ProjectService"/>). Older records have none.</param>
public sealed record ProjectRecord(Guid Id, Guid SequenceId, string Name, bool IsMain, ProjectHead? Head, DateTimeOffset Created, string? CreatedBy,
DateTimeOffset Updated, DateTimeOffset? Deleted = null)
DateTimeOffset Updated, DateTimeOffset? Deleted = null, ProjectHead? Mirrored = null)
{
/// <summary>The record of an index row.</summary>
public static ProjectRecord Of(SequenceProject project)
/// <summary>The record of an index row, naming <paramref name="mirrored"/> as the version mirrored last (Main).</summary>
public static ProjectRecord Of(SequenceProject project, ProjectHead? mirrored = null)
{
ArgumentNullException.ThrowIfNull(project);
return new ProjectRecord(project.Id, project.SequenceId, project.Name, project.IsMain,
project.HeadVersionId is { } head ? new ProjectHead(head, project.HeadNumber) : null, project.CreatedAt, project.CreatedBy, project.UpdatedAt,
project.DeletedAt);
project.DeletedAt, project.IsMain ? mirrored : null);
}
/// <summary>The index row of this record (nothing mirrored known yet).</summary>
@@ -58,20 +64,33 @@ public static class ProjectSidecars
public const string VersionFormat = "chronos-project-version";
public const int FormatVersion = 1;
/// <summary>A version file is a few kilobytes; a larger one is not read.</summary>
public const int MaxVersionBytes = 8 << 20;
/// <summary>
/// The one size limit of a version file, for writing and reading alike: a version that could not be read back is never
/// written (<see cref="WriteVersion"/>). A version is a few kilobytes; each photo it parks (an exclusion, the hero, a
/// keyframe waiting for its photo) takes about 340 bytes of the file, so this holds the positions of about 190,000
/// photos, well past the largest sequences.
/// </summary>
public const int MaxVersionBytes = 64 << 20;
private static readonly JsonWriterOptions Indented = new() { Indented = true };
/// <summary>Whether <paramref name="version"/> can be stored: its file within <see cref="MaxVersionBytes"/>.</summary>
public static bool Fits(ProjectVersion version) => Encoding.UTF8.GetByteCount(VersionJson(version)) <= MaxVersionBytes;
/// <summary>
/// Writes a version file. A version is immutable: an existing file is left when it holds the same version (same id
/// and content hash) and refused otherwise, unless <paramref name="replace"/> (a workspace's own copy of a version,
/// stored at its home since).
/// </summary>
/// <exception cref="IOException">Another version is at that path.</exception>
/// <exception cref="ProjectVersionTooLargeException">The file would be over <see cref="MaxVersionBytes"/>: nothing was written.</exception>
public static void WriteVersion(string path, ProjectVersion version, bool replace = false)
{
ArgumentNullException.ThrowIfNull(version);
var json = VersionJson(version);
var bytes = Encoding.UTF8.GetByteCount(json);
if (bytes > MaxVersionBytes)
throw new ProjectVersionTooLargeException($"Version {version.Number} of project {version.ProjectId} takes {bytes} bytes, over the {MaxVersionBytes} a version may.");
if (!replace && File.Exists(path))
{
ProjectVersion existing;
@@ -92,11 +111,11 @@ public static class ProjectSidecars
}
}
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
AtomicFile.WriteAllText(path, VersionJson(version));
AtomicFile.WriteAllText(path, json);
}
/// <summary>Reads a version file and checks its content hash.</summary>
/// <exception cref="InvalidDataException">Not a version file, damaged, newer, or its settings do not match their hash.</exception>
/// <exception cref="InvalidDataException">Not a version file, damaged, newer, over <see cref="MaxVersionBytes"/>, or its settings do not match their hash.</exception>
/// <exception cref="IOException">The file cannot be read.</exception>
public static ProjectVersion ReadVersion(string path)
{
@@ -189,6 +208,13 @@ public static class ProjectSidecars
if (record.CreatedBy is { } by) w.WriteString("createdBy", by);
w.WriteString("updated", record.Updated);
if (record.Deleted is { } deleted) w.WriteString("deleted", deleted);
if (record.Mirrored is { } mirrored)
{
w.WriteStartObject("mirrored");
w.WriteString("id", mirrored.Id);
w.WriteNumber("number", mirrored.Number);
w.WriteEndObject();
}
w.WriteEndObject();
}
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
@@ -203,22 +229,26 @@ public static class ProjectSidecars
using var document = Parse(AtomicFile.ReadAllText(path), path);
var root = document.RootElement;
Check(root, RecordFormat, path);
ProjectHead? head = null;
if (root.TryGetProperty("head", out var h) && h.ValueKind == JsonValueKind.Object && Id(h, "id") is { } headId && Int(h, "number") is { } number and > 0)
head = new ProjectHead(headId, number);
var name = Text(root, "name");
return new ProjectRecord(
Id(root, "id") ?? throw new InvalidDataException($"{path} has no project id."),
Id(root, "sequenceId") ?? throw new InvalidDataException($"{path} names no sequence."),
string.IsNullOrWhiteSpace(name) ? throw new InvalidDataException($"{path} has no name.") : name,
root.TryGetProperty("isMain", out var main) && main.ValueKind == JsonValueKind.True,
head,
Head(root, "head"),
Time(root, "created") ?? default,
Text(root, "createdBy"),
Time(root, "updated") ?? Time(root, "created") ?? default,
Time(root, "deleted"));
Time(root, "deleted"),
Head(root, "mirrored"));
}
/// <summary>A version a record names (its head, the version mirrored): <c>{ "id", "number" }</c>; null when it names none.</summary>
private static ProjectHead? Head(JsonElement root, string name) =>
root.TryGetProperty(name, out var h) && h.ValueKind == JsonValueKind.Object && Id(h, "id") is { } id && Int(h, "number") is { } number and > 0
? new ProjectHead(id, number)
: null;
/// <summary>The version files of a folder by number and id (<c>000001-&lt;id&gt;.json</c>), in number order; other files are not listed.</summary>
public static IReadOnlyList<(int Number, Guid Id, string Path)> ListVersions(string directory)
{
@@ -0,0 +1,502 @@
using System.Security.Cryptography;
using Chronos.Core.Archiving;
using Chronos.Core.Model;
using Chronos.Core.Project;
using Chronos.Data.Archiving;
using Chronos.Data.Projects;
using Chronos.Data.Tests.Archiving;
using Chronos.Data.Tests.Infrastructure;
using Chronos.Testing;
using static Chronos.Data.Tests.Archiving.ArchiveFixture;
using static Chronos.Data.Tests.Projects.ProjectServiceTests;
namespace Chronos.Data.Tests.Projects;
/// <summary>
/// The sidecars stay the truth of a sequence's projects when they cannot be read, are missing, are a newer Chronos's, or
/// sit on a read-only archive, and an index that meets them late never takes its own old copy for the sequence's settings.
/// </summary>
[Category("projects")]
public abstract class ProjectRecoveryTests(TestContext context) : DatabaseTest(context)
{
private async Task<ArchiveFixture> FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
/// <summary>The live project folders of a sequence (one per project).</summary>
private static string[] ProjectFolders(SequencePaths folder) =>
[.. Directory.GetDirectories(folder.ProjectsDirectory).Where(d => Guid.TryParseExact(Path.GetFileName(d), "N", out _))];
private static int MainRecords(SequencePaths folder) =>
ProjectFolders(folder).Count(d => ProjectSidecars.ReadRecord(Path.Combine(d, "project.json")).IsMain);
private static ProjectSettings Settings(ArchiveFixture fx, Sequence sequence) =>
ProjectSettings.FromJson(File.ReadAllText(fx.Paths.Sequence(sequence.Folder).ProjectPath));
[Test]
public async Task ARecordThatCannotBeReadNeverTakesItsProjectForGoneNorMakesASecondMain()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
var mainRecord = File.ReadAllText(folder.ProjectRecordPath(main.Id));
var verticalRecord = File.ReadAllText(folder.ProjectRecordPath(vertical.Id));
// Main's record as a newer Chronos writes it, Vertical's damaged.
var newer = mainRecord.Replace("\"formatVersion\": 1", "\"formatVersion\": 2", StringComparison.Ordinal);
File.WriteAllText(folder.ProjectRecordPath(main.Id), newer);
File.WriteAllText(folder.ProjectRecordPath(vertical.Id), "{ not json");
Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id);
Assert.Count(2, ProjectFolders(folder));
Assert.Null((await fx.Db.Projects.GetAsync(vertical.Id))!.DeletedAt);
// Nothing is written over a record that cannot be read: the versions sent wait.
var head = await projects.HeadAsync(vertical.Id, fx.Ct);
await Assert.ThrowsAsync<ProjectsUnreadableException>(() => projects.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct));
var home = new ArchiveProjectHome(fx.Db, projects, sequence.Id);
await Assert.ThrowsAsync<HomeUnavailableException>(() => home.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct));
await Assert.ThrowsAsync<ProjectsUnreadableException>(() => projects.RenameAsync(vertical.Id, "Square", fx.Ct));
await Assert.ThrowsAsync<ProjectsUnreadableException>(() => projects.DeleteAsync(vertical.Id, fx.Ct));
Assert.Equal("{ not json", File.ReadAllText(folder.ProjectRecordPath(vertical.Id)));
Assert.Equal(newer, File.ReadAllText(folder.ProjectRecordPath(main.Id)));
// Readable again: everything is as it was, and goes on.
File.WriteAllText(folder.ProjectRecordPath(main.Id), mainRecord);
File.WriteAllText(folder.ProjectRecordPath(vertical.Id), verticalRecord);
Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
// An index that meets Main's record unreadable makes no Main of its own either, and takes it once it reads.
File.WriteAllText(folder.ProjectRecordPath(main.Id), newer);
await fx.ReopenWithEmptyIndexAsync();
Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
var rebuilt = fx.Service.Projects;
Assert.SequenceEqual([vertical.Id], (await rebuilt.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
await Assert.ThrowsAsync<ProjectsUnreadableException>(() => rebuilt.MainAsync(sequence.Id, fx.Ct));
await Assert.ThrowsAsync<HomeUnavailableException>(() => new ArchiveProjectHome(fx.Db, rebuilt, sequence.Id).ListAsync(fx.Ct));
Assert.Count(2, ProjectFolders(folder));
File.WriteAllText(folder.ProjectRecordPath(main.Id), mainRecord);
Assert.Equal(main.Id, (await rebuilt.MainAsync(sequence.Id, fx.Ct)).Id);
Assert.Equal(1, MainRecords(folder));
}
[Test]
public async Task AMissingProjectsFolderIsWrittenAgainFromTheIndex()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var pushed = (await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [2] }, frames),
fx.Ct)).Version!;
var folder = fx.Paths.Sequence(sequence.Folder);
// Moved away for a moment (a sync tool, a backup put back without it): no project is taken for gone, none made anew.
Directory.Move(folder.ProjectsDirectory, folder.ProjectsDirectory + "-aside");
Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
Assert.Equal(1, MainRecords(folder));
Assert.Equal(pushed.Id, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Head!.Id);
Assert.Equal(pushed.ContentHash, ProjectSidecars.ReadVersion(folder.VersionPath(vertical.Id, 2, pushed.Id)).ContentHash);
Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(pushed, new ProjectSettings(), frames), fx.Ct)).Outcome);
// The sidecars written again are the truth again: a rebuilt index has every project and version back.
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct);
Assert.SequenceEqual([main.Id, vertical.Id], listed.Select(p => p.Id));
Assert.Equal(3, listed[1].HeadNumber);
}
[Test]
public async Task AMainAnOlderChronosTookForGoneIsListedAgain()
{
var fx = await FixtureAsync();
var (sequence, _) = await UploadedAsync(fx, 2);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
// Its live record says it is not deleted: Main again, no second one.
await fx.Db.Projects.UpdateAsync((await fx.Db.Projects.GetAsync(main.Id))! with { DeletedAt = DateTimeOffset.UtcNow });
Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id);
// And with its folder gone too: written again from the index.
await fx.Db.Projects.UpdateAsync((await fx.Db.Projects.GetAsync(main.Id))! with { DeletedAt = DateTimeOffset.UtcNow });
Directory.Move(folder.ProjectDirectory(main.Id), folder.ProjectDirectory(main.Id) + "-aside");
Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
Assert.True(ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).IsMain);
Assert.Equal(1, MainRecords(folder));
}
[Test]
public async Task ASecondIndexMeetingMainBetweenTwoPushesRevertsNothing()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var second = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct),
new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 0.7 } }, frames), fx.Ct)).Version!;
// Another PC indexes the archive (its copy of the settings: version 2's), then this one pushes version 3.
var (other, _) = await fx.OtherWriterAsync();
var third = (await projects.PushAsync(main.Id, Draft(second, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 1.4 } }, frames),
fx.Ct)).Version!;
// Its first look takes the settings from the archive, not from its old copy: no version is added.
await other.Projects.ListAsync(sequence.Id, fx.Ct);
Assert.SequenceEqual([3, 2, 1], (await other.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number));
Assert.Equal(1.4, Settings(fx, sequence).Exposure.ExposureOffsetEv);
Assert.Equal(third.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id);
Assert.Equal(1.4, ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).Exposure.ExposureOffsetEv);
}
[Test]
public async Task AnIndexWithAnOldCopyOfTheSettingsBringsItUpToMain()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var pushed = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [0] }, frames),
fx.Ct)).Version!;
// An index taken back from a backup made before: it holds the sequence with the settings it had then, and no projects.
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync();
await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [1] });
var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct);
Assert.Equal((pushed.Id, 2), (listed[0].HeadVersionId, listed[0].HeadNumber));
Assert.SequenceEqual([2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number));
Assert.SequenceEqual([0], ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).ExcludedFrames);
Assert.SequenceEqual([0], Settings(fx, sequence).ExcludedFrames);
}
[Test]
public async Task AMirrorACrashCutShortBeforeProjectJsonIsCompletedAfterARebuild()
{
foreach (var olderRecord in new[] { false, true })
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var first = await projects.HeadAsync(main.Id, fx.Ct);
await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.Median } }, frames), fx.Ct);
// As if the push had stopped before its mirror wrote project.json (an older record names no mirror at all).
var folder = fx.Paths.Sequence(sequence.Folder);
File.WriteAllText(folder.ProjectPath, ProjectSettings.FromJson(ProjectContent.Place(first.SettingsJson, Photos(frames))).ToJson());
var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id));
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Mirrored = olderRecord ? null : new ProjectHead(first.Id, 1) });
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
Assert.SequenceEqual([2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number));
Assert.Equal(StackSetting.Median, Settings(fx, sequence).Stack.Mode);
Assert.Equal(StackSetting.Median, ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).Stack.Mode);
}
}
[Test]
public async Task SettingsAnOlderChronosWroteIntoProjectJsonBecomeAnExternalVersionForAnIndexMeetingMain()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 0.7 } },
frames), fx.Ct);
// An older Chronos on another PC edits the sequence (project.json); this PC's index is then rebuilt.
File.WriteAllText(fx.Paths.Sequence(sequence.Folder).ProjectPath, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = -1 } }.ToJson());
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
var history = await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct);
Assert.SequenceEqual([3, 2, 1], history.Select(v => v.Number));
Assert.Equal(ProjectVersionKind.External, history[0].Kind);
Assert.Equal(-1, ProjectSettings.FromJson(history[0].SettingsJson).Exposure.ExposureOffsetEv);
}
[Test]
public async Task AMainHeadOfANewerChronosIsListedAndPulledButNeverMirroredNorBuiltOn()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var head = await projects.HeadAsync(main.Id, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
// A newer Chronos on another PC sharing the archive pushes version 2 in its settings format, and mirrors it.
var settings = """{ "version": 2, "exposure": { "exposureOffsetEv": 0.5 }, "futureThing": true }""";
var now = DateTimeOffset.UtcNow;
var future = new ProjectVersion
{
Id = Guid.CreateVersion7(),
ProjectId = main.Id,
ParentId = head.Id,
Number = 2,
Kind = ProjectVersionKind.Commit,
ClientName = "newer",
SettingsJson = settings,
ContentHash = ProjectContent.Hash(settings),
SettingsFormat = 2,
FramesVersion = head.FramesVersion,
CreatedAt = now,
StoredAt = now,
};
ProjectSidecars.WriteVersion(folder.VersionPath(main.Id, 2, future.Id), future);
var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id));
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Head = new ProjectHead(future.Id, 2), Mirrored = new ProjectHead(future.Id, 2) });
File.WriteAllText(folder.ProjectPath, settings);
var copy = (await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson;
Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
Assert.Equal(future.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id);
Assert.Equal(future.Id, (await new ArchiveProjectHome(fx.Db, projects, sequence.Id).VersionsAfterAsync(main.Id, 1, 10, fx.Ct)).Single().Id);
Assert.Equal(PushOutcome.Stored,
(await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
Assert.Count(2, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
Assert.Equal(settings, File.ReadAllText(folder.ProjectPath));
Assert.Equal(copy, (await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson);
// An older path writes the settings here meanwhile: nothing is merged into, or added over, the newer head.
await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [2] });
await projects.ListAsync(sequence.Id, fx.Ct);
await projects.ListAsync(sequence.Id, fx.Ct);
Assert.Count(2, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
Assert.Equal(settings, File.ReadAllText(folder.ProjectPath));
// Nor by an index meeting it for the first time.
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
Assert.Equal(future.Id, (await fx.Service.Projects.HeadAsync(main.Id, fx.Ct)).Id);
Assert.Count(2, await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
Assert.Equal(settings, File.ReadAllText(folder.ProjectPath));
}
[Test]
public async Task APushThisChronosCannotReadStrictlyIsRefused()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 2);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var head = await projects.HeadAsync(main.Id, fx.Ct);
var wrong = """{ "exposure": 5 }""";
var result = await projects.PushAsync(main.Id, Draft(head, new ProjectSettings(), frames) with { SettingsJson = wrong, ContentHash = ProjectContent.Hash(wrong) },
fx.Ct);
Assert.Equal(PushOutcome.Invalid, result.Outcome);
Assert.Count(1, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(main.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
}
[Test]
public async Task AHeadThisChronosCannotReadLeavesTheRecordAndTheHistoryUntouched()
{
foreach (var missing in new[] { false, true })
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var first = await projects.HeadAsync(main.Id, fx.Ct);
var second = (await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!;
var folder = fx.Paths.Sequence(sequence.Folder);
// Version 2's file in a newer Chronos's format, or not here yet (a sync tool that brought the record first).
var path = folder.VersionPath(main.Id, 2, second.Id);
var text = File.ReadAllText(path);
if (missing) File.Delete(path);
else File.WriteAllText(path, text.Replace("\"formatVersion\": 1", "\"formatVersion\": 2", StringComparison.Ordinal));
var recordBytes = File.ReadAllBytes(folder.ProjectRecordPath(main.Id));
await fx.ReopenWithEmptyIndexAsync();
Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
var rebuilt = fx.Service.Projects;
var listed = await rebuilt.ListAsync(sequence.Id, fx.Ct);
Assert.Equal((main.Id, first.Id), (listed.Single().Id, listed.Single().HeadVersionId));
Assert.Equal(first.Id, (await rebuilt.HeadAsync(main.Id, fx.Ct)).Id);
await Assert.ThrowsAsync<ProjectsUnreadableException>(() => rebuilt.PushAsync(main.Id, Draft(first, new ProjectSettings(), frames), fx.Ct));
Assert.SequenceEqual(recordBytes, File.ReadAllBytes(folder.ProjectRecordPath(main.Id)));
Assert.False(Directory.Exists(Path.Combine(folder.ProjectVersionsDirectory(main.Id), ".stray")));
Assert.Count(missing ? 1 : 2, ProjectSidecars.ListVersions(folder.ProjectVersionsDirectory(main.Id)));
Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
// Once it reads, the history goes on from it.
File.WriteAllText(path, text);
Assert.Equal(second.Id, (await rebuilt.HeadAsync(main.Id, fx.Ct)).Id);
Assert.Equal(PushOutcome.Stored, (await rebuilt.PushAsync(main.Id, Draft(second, new ProjectSettings(), frames), fx.Ct)).Outcome);
}
}
[Test]
public async Task VersionFilesOfOneNumberAndRecordsThatAllSayMainNeverStopARebuild()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 3);
var other = (await UploadedAsync(fx, 1, seed: 300)).Sequence;
var projects = fx.Service.Projects;
var main = await projects.MainAsync(sequence.Id, fx.Ct);
var otherMain = await projects.MainAsync(other.Id, fx.Ct);
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var first = await projects.HeadAsync(vertical.Id, fx.Ct);
var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct)).Version!;
var folder = fx.Paths.Sequence(sequence.Folder);
// Another file numbered 2, off the record's chain, and two more records that say they are Main and are named so.
var twin = Draft(first, new ProjectSettings { ExcludedFrames = [0] }, frames) with { StoredAt = DateTimeOffset.UtcNow };
ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 2, twin.Id), twin);
foreach (var _ in new[] { 1, 2 })
{
var id = Guid.CreateVersion7();
ProjectSidecars.WriteRecord(folder.ProjectRecordPath(id), new ProjectRecord(id, sequence.Id, ProjectService.MainName, true, null, DateTimeOffset.UtcNow, null,
DateTimeOffset.UtcNow));
}
await fx.ReopenWithEmptyIndexAsync();
Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct);
Assert.Equal(main.Id, listed.Single(p => p.IsMain).Id);
Assert.SequenceEqual(["Main", "Main (2)", "Main (3)", "Vertical"], listed.Select(p => p.Name).Order(StringComparer.Ordinal));
Assert.Equal(second.Id, listed.Single(p => p.Id == vertical.Id).HeadVersionId);
Assert.Count(1, Directory.GetFiles(Path.Combine(folder.ProjectVersionsDirectory(vertical.Id), ".stray")));
Assert.NotNull(await fx.Db.Projects.GetAsync(otherMain.Id));
}
[Test]
public async Task ALargeVersionIsStoredAndReadBackAndOneTooLargeIsNeverStored()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 2);
var projects = fx.Service.Projects;
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var head = await projects.HeadAsync(vertical.Id, fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
// Some 30,000 photos parked by a long sequence (its nights excluded): past the old 8 MB read limit.
var pending = Enumerable.Range(0, 30_000).Select(i => new PendingFrame
{
Sha256 = Convert.ToHexStringLower(SHA256.HashData(BitConverter.GetBytes(i))),
Size = 1_000_000 + i,
CapturedAt = Start.AddSeconds(i),
Excluded = true,
}).ToList();
var settings = ProjectContent.Canonical(new ProjectSettings { PendingFrames = pending }.ToJson());
var large = Draft(head, new ProjectSettings(), frames) with { SettingsJson = settings, ContentHash = ProjectContent.Hash(settings) };
Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, large, fx.Ct)).Outcome);
var path = folder.VersionPath(vertical.Id, 2, large.Id);
Assert.Greater(new FileInfo(path).Length, 8L << 20);
Assert.Equal(large.ContentHash, ProjectSidecars.ReadVersion(path).ContentHash);
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync(ScanMode.Rebuild);
var rebuilt = fx.Service.Projects;
Assert.Equal(large.Id, (await rebuilt.HeadAsync(vertical.Id, fx.Ct)).Id);
// One whose file would be past the limit is refused, and nothing is written.
var huge = Draft(large, new ProjectSettings(), frames) with { Note = new string('x', ProjectSidecars.MaxVersionBytes) };
Assert.Equal(PushOutcome.TooLarge, (await rebuilt.PushAsync(vertical.Id, huge, fx.Ct)).Outcome);
Assert.False(File.Exists(folder.VersionPath(vertical.Id, 3, huge.Id)));
Assert.Throws<ProjectVersionTooLargeException>(() => ProjectSidecars.WriteVersion(Path.Combine(Context.CreateTempDirectory("huge"), "v.json"), huge));
}
[Test]
public async Task TheHomeHoldsAVersionOnlyWhileItsFileReadsBack()
{
var fx = await FixtureAsync();
var (sequence, frames) = await UploadedAsync(fx, 2);
var projects = fx.Service.Projects;
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var first = await projects.HeadAsync(vertical.Id, fx.Ct);
var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!;
var folder = fx.Paths.Sequence(sequence.Folder);
IProjectHome home = new ArchiveProjectHome(fx.Db, projects, sequence.Id);
Assert.True(await home.HoldsAsync(vertical.Id, first, fx.Ct));
Assert.True(await home.HoldsAsync(vertical.Id, second, fx.Ct));
Assert.False(await home.HoldsAsync(vertical.Id, second with { ContentHash = new string('0', 64) }, fx.Ct));
// The index still has both rows, but the archive no longer holds them as they were.
File.WriteAllText(folder.VersionPath(vertical.Id, 1, first.Id), "damaged");
File.Delete(folder.VersionPath(vertical.Id, 2, second.Id));
Assert.NotNull(await home.GetVersionAsync(vertical.Id, first.Id, fx.Ct));
Assert.False(await home.HoldsAsync(vertical.Id, first, fx.Ct));
Assert.False(await home.HoldsAsync(vertical.Id, second, fx.Ct));
}
[Test]
public async Task ADeleteThatCannotMoveTheFolderChangesNothing()
{
var fx = await FixtureAsync();
var (sequence, _) = await UploadedAsync(fx, 2);
var projects = fx.Service.Projects;
var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
var folder = fx.Paths.Sequence(sequence.Folder);
// The projects' trash cannot take the folder (a file in its way): the delete fails and says so, and nothing changed.
File.WriteAllText(folder.ProjectsTrashDirectory, "in the way");
await Assert.ThrowsAsync<IOException>(() => projects.DeleteAsync(vertical.Id, fx.Ct));
Assert.Null(ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Deleted);
Assert.SequenceEqual(["Main", "Vertical"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name));
File.Delete(folder.ProjectsTrashDirectory);
await projects.DeleteAsync(vertical.Id, fx.Ct);
Assert.False(Directory.Exists(folder.ProjectDirectory(vertical.Id)));
Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(), "project.json")).Deleted);
Assert.SequenceEqual(["Main"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name));
}
[Test]
public async Task ASequenceOfAReadOnlyArchiveOpensFromItsSidecarsWithoutWriting()
{
var fx = await FixtureAsync();
var (fresh, freshFrames) = await UploadedAsync(fx, 2);
var (edited, frames) = await UploadedAsync(fx, 2, seed: 300);
var vertical = await fx.Service.Projects.CreateAsync(edited.Id, "Vertical", cancellationToken: fx.Ct);
var pushed = (await fx.Service.Projects.PushAsync(vertical.Id,
Draft(await fx.Service.Projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!;
await fx.ReopenWithEmptyIndexAsync();
await fx.ScanAsync();
var projects = fx.Service.Projects;
projects.Availability = _ => ArchiveAvailability.ReadOnly;
var locks = Directory.Exists(fx.Paths.LocksDirectory) ? Directory.GetFiles(fx.Paths.LocksDirectory).Length : 0;
// A sequence with projects: read from its sidecars as they are.
var home = new ArchiveProjectHome(fx.Db, projects, edited.Id);
Assert.SequenceEqual(["Main", "Vertical"], (await home.ListAsync(fx.Ct)).Select(p => p.Name));
Assert.Equal(pushed.Id, (await home.HeadAsync(vertical.Id, fx.Ct)).Id);
await Assert.ThrowsAsync<HomeUnavailableException>(() => home.PushAsync(vertical.Id, Draft(pushed, new ProjectSettings(), frames), fx.Ct));
// A sequence with none yet: a Main in the index only, nothing written to the archive.
var freshHome = new ArchiveProjectHome(fx.Db, projects, fresh.Id);
var main = (await freshHome.ListAsync(fx.Ct)).Single();
Assert.True(main.IsMain);
var head = await freshHome.HeadAsync(main.Id, fx.Ct);
Assert.False(Directory.Exists(fx.Paths.Sequence(fresh.Folder).ProjectsDirectory));
Assert.Equal(locks, Directory.Exists(fx.Paths.LocksDirectory) ? Directory.GetFiles(fx.Paths.LocksDirectory).Length : 0);
await Assert.ThrowsAsync<HomeUnavailableException>(() => freshHome.PushAsync(main.Id, Draft(head, new ProjectSettings(), freshFrames), fx.Ct));
// Once the archive takes writes, that Main's sidecars are written, under the same id, and it goes on.
projects.Availability = ArchiveVolume.Probe;
Assert.Equal(main.Id, (await freshHome.ListAsync(fx.Ct)).Single().Id);
var folder = fx.Paths.Sequence(fresh.Folder);
Assert.Equal((main.Id, true), (ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).Id, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).IsMain));
Assert.FileExists(folder.VersionPath(main.Id, 1, head.Id));
Assert.Equal(PushOutcome.Stored, (await freshHome.PushAsync(main.Id, Draft(head, new ProjectSettings { ExcludedFrames = [0] }, freshFrames), fx.Ct)).Outcome);
}
}
public sealed class SqliteProjectRecoveryTests(TestContext context) : ProjectRecoveryTests(context)
{
protected override Backend Backend => Backend.SqliteFile;
}
[RequiresEnvironment(PgTestServer.Variable)]
public sealed class PostgresProjectRecoveryTests(TestContext context) : ProjectRecoveryTests(context)
{
protected override Backend Backend => Backend.Postgres;
}