diff --git a/src/Chronos.App/Workspace/WorkspaceHost.cs b/src/Chronos.App/Workspace/WorkspaceHost.cs
index 68ce5fd3..7290f506 100644
--- a/src/Chronos.App/Workspace/WorkspaceHost.cs
+++ b/src/Chronos.App/Workspace/WorkspaceHost.cs
@@ -491,10 +491,8 @@ public sealed class WorkspaceHost
var removed = 0;
foreach (var version in stored.Where(v => v.Id != basis?.VersionId && !kept.Contains(v.Id)))
{
- if (await home.GetVersionAsync(projectId, version.Id, cancellationToken).ConfigureAwait(false) is not { } atHome
- || !string.Equals(atHome.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase)
- || !ProjectContent.Verifies(atHome.SettingsJson, atHome.ContentHash))
- continue;
+ // Read back from where the home keeps it, not from an index of it.
+ if (!await home.HoldsAsync(projectId, version, cancellationToken).ConfigureAwait(false)) continue;
File.Delete(project.VersionPath(version));
removed++;
}
@@ -528,16 +526,20 @@ public sealed class WorkspaceHost
var home = HomeOf(workspace.Home) ?? throw new WorkspaceException(Loc.L("Errors.Workspace.Offline"));
ProjectVersion head;
ProjectVersion? basis;
+ bool held;
try
{
head = await home.HeadAsync(projectId, cancellationToken).ConfigureAwait(false);
basis = workspace.Base is { } b ? await home.GetVersionAsync(projectId, b.VersionId, cancellationToken).ConfigureAwait(false) : head;
+ // Read back from where the home keeps them, not from an index of them.
+ held = basis is not null && await home.HoldsAsync(projectId, head, cancellationToken).ConfigureAwait(false)
+ && (basis.Id == head.Id || await home.HoldsAsync(projectId, basis, cancellationToken).ConfigureAwait(false));
}
catch (Exception ex) when (ex is HomeUnavailableException or ProjectException or DatabaseException)
{
throw new WorkspaceException(Loc.L("Errors.Workspace.Offline"), ex);
}
- if (!ProjectContent.Verifies(head.SettingsJson, head.ContentHash) || basis is null || !ProjectContent.Verifies(basis.SettingsJson, basis.ContentHash)
+ if (!held || !ProjectContent.Verifies(head.SettingsJson, head.ContentHash) || basis is null || !ProjectContent.Verifies(basis.SettingsJson, basis.ContentHash)
|| (workspace.Base is { } known && !string.Equals(known.ContentHash, basis.ContentHash, StringComparison.OrdinalIgnoreCase)))
throw new WorkspaceException(Loc.L("Errors.Workspace.HomeCheck"));
}
diff --git a/src/Chronos.Data/Projects/IProjectHome.cs b/src/Chronos.Data/Projects/IProjectHome.cs
index 15a30f85..098ec3b8 100644
--- a/src/Chronos.Data/Projects/IProjectHome.cs
+++ b/src/Chronos.Data/Projects/IProjectHome.cs
@@ -33,6 +33,18 @@ public interface IProjectHome
/// A version of the project by id; null when the home has none.
Task GetVersionAsync(Guid projectId, Guid versionId, CancellationToken cancellationToken);
+ ///
+ /// Whether the home holds durably, read back from where it keeps it and matching its content
+ /// hash: what a workspace asks before it drops its own copy. A home that keeps no other copy answers from
+ /// .
+ ///
+ async Task HoldsAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken)
+ {
+ ArgumentNullException.ThrowIfNull(version);
+ return await GetVersionAsync(projectId, version.Id, cancellationToken).ConfigureAwait(false) is { } held
+ && string.Equals(held.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase) && ProjectContent.Verifies(held.SettingsJson, held.ContentHash);
+ }
+
/// The versions numbered above , oldest first, at most .
Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken);
@@ -91,6 +103,16 @@ public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projec
return version is not null && version.ProjectId == projectId ? version : null;
});
+ /// Read back from the version's file in the archive, never from the index's row ( ).
+ public Task HoldsAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken) =>
+ Reach(async () =>
+ {
+ ArgumentNullException.ThrowIfNull(version);
+ var project = await Own(projectId, cancellationToken).ConfigureAwait(false);
+ return project.DeletedAt is null && ProjectContent.Verifies(version.SettingsJson, version.ContentHash)
+ && await projects.StoredAsync(projectId, version, cancellationToken).ConfigureAwait(false);
+ });
+
public Task> VersionsAfterAsync(Guid projectId, int afterNumber, int limit, CancellationToken cancellationToken) =>
Reach(async () =>
{
@@ -151,5 +173,10 @@ public sealed class ArchiveProjectHome(ChronosDatabase db, ProjectService projec
// Another index of the archive moved the sequence: this one finds it at its next scan.
throw new HomeUnavailableException(ex.Message, ex);
}
+ catch (ProjectsUnreadableException ex)
+ {
+ // A record or a version this Chronos cannot read now (damaged, held open, of a newer Chronos): nothing was written.
+ throw new HomeUnavailableException(ex.Message, ex);
+ }
}
}
diff --git a/src/Chronos.Data/Projects/ProjectIndexer.cs b/src/Chronos.Data/Projects/ProjectIndexer.cs
index aaf75ab6..84401b38 100644
--- a/src/Chronos.Data/Projects/ProjectIndexer.cs
+++ b/src/Chronos.Data/Projects/ProjectIndexer.cs
@@ -5,50 +5,97 @@ using Chronos.Core.Model;
namespace Chronos.Data.Projects;
+/// What a reconcile of a sequence's projects met ( ).
+/// The sequence has no projects folder: nothing was read, and nothing taken for gone.
+/// The projects the archive has a folder for, live or in the projects' trash, whether or not its record read.
+/// The live projects the index could not be brought in line with: a record that cannot be read (damaged,
+/// of a newer Chronos, a read failure), a head whose version this Chronos cannot read, or a row the index refused. Each stays
+/// as the index has it, and nothing is written for it until it reads again.
+/// The records of the live projects read, by id (their head as moved forward).
+internal sealed record ProjectReconcile(bool Missing, IReadOnlySet Seen, IReadOnlySet HeldBack, IReadOnlyDictionary Live)
+{
+ /// Every live project of the folder was read and indexed: a sequence without a Main may be given one.
+ public bool Complete => HeldBack.Count == 0;
+}
+
///
/// Brings the index's projects of one sequence in line with their sidecars (owner review 2, R8), which are the truth: a
/// project or version the index lacks is added (an index rebuilt, a sequence restored from the trash), a head the index is
-/// behind on moves forward, a name, Main flag or deletion recorded since is taken, and a live index row whose folder is gone
-/// is marked deleted. A version file written past its record's head by a push a crash cut short is adopted when it
-/// follows the head (its file is complete: versions are written atomically, then the record); any other file past the
-/// head is moved aside to versions/.stray , never deleted. Runs under the sequence's lease.
+/// behind on moves forward, a name or deletion recorded since is taken, and a row the index marked deleted is listed again
+/// when its live record says it is not. Only a record that says so, live or in the projects' trash, deletes a project:
+/// never a folder that is not there (moved for a moment, a backup put back, a sync tool) nor a record that cannot be read,
+/// whose project stays as the index has it ( ). A version file written past its
+/// record's head by a push a crash cut short is adopted when it follows the head (its file is complete: versions are
+/// written atomically, then the record); any other file past the head, or a second file of a number the record's chain
+/// names another for, is moved aside to versions/.stray , never deleted. Runs under the sequence's lease, or without
+/// writing to the archive at all when it is read-only.
///
internal sealed class ProjectIndexer(ChronosDatabase db)
{
private const string StrayFolder = ".stray";
/// Reconciles the projects of in ; the caller holds the sequence's lease.
- public async Task ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken)
+ /// False for a read-only archive: the sidecars are read as they are and only the index is written
+ /// (a version past its record's head is indexed, not adopted into the record; nothing is moved aside).
+ public async Task ReconcileAsync(SequencePaths folder, Guid sequenceId, CancellationToken cancellationToken, bool writable = true)
{
var seen = new HashSet();
- if (Directory.Exists(folder.ProjectsDirectory))
+ var heldBack = new HashSet();
+ var live = new Dictionary();
+ if (!Directory.Exists(folder.ProjectsDirectory)) return new ProjectReconcile(true, seen, heldBack, live);
+ foreach (var (directory, trashed) in ProjectFolders(folder))
{
- foreach (var (directory, trashed) in ProjectFolders(folder))
+ cancellationToken.ThrowIfCancellationRequested();
+ ProjectRecord record;
+ try
{
- cancellationToken.ThrowIfCancellationRequested();
- ProjectRecord record;
- try
- {
- record = ProjectSidecars.ReadRecord(Path.Combine(directory, ArchiveLayout.ProjectFile));
- }
- catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
- {
- Log.Warning("projects", $"The project in {directory} cannot be read", ex);
- continue;
- }
- // A sequence folder copied by hand carries the projects of its source, which stay that sequence's.
- if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue;
- if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) };
- if (record.Deleted is null) record = RollForward(folder, record, directory);
- await IndexAsync(record, directory, cancellationToken).ConfigureAwait(false);
+ record = ProjectSidecars.ReadRecord(Path.Combine(directory, ArchiveLayout.ProjectFile));
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ Log.Warning("projects", $"The project in {directory} cannot be read; it stays as the index has it", ex);
+ // Its folder names it (, or - in the trash): its row stays as it is, never taken for gone nor written
+ // again, and a live one is held back (it may be Main).
+ var name = Path.GetFileName(directory);
+ if (Guid.TryParseExact(trashed && name.Length > 32 ? name[..32] : name, "N", out var named) && seen.Add(named) && !trashed) heldBack.Add(named);
+ continue;
+ }
+ // A sequence folder copied by hand carries the projects of its source, which stay that sequence's.
+ if (record.SequenceId != sequenceId || !seen.Add(record.Id)) continue;
+ if (trashed && record.Deleted is null) record = record with { Deleted = Directory.GetLastWriteTimeUtc(directory) };
+ if (record.Deleted is null)
+ {
+ record = RollForward(folder, record, directory, writable);
+ live[record.Id] = record;
+ }
+ try
+ {
+ if (!await IndexAsync(record, directory, trashed, writable, cancellationToken).ConfigureAwait(false)) heldBack.Add(record.Id);
+ }
+ catch (DatabaseException ex) when (ex.IsUniqueViolation || ex.IsForeignKeyViolation)
+ {
+ // One project the index refuses never stops the others, nor a scan of the archive: it is read again next time.
+ Log.Warning("projects", $"Project {record.Id} of sequence {sequenceId} could not be indexed", ex);
+ if (record.Deleted is null) heldBack.Add(record.Id);
}
}
- foreach (var gone in await db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
+ return new ProjectReconcile(false, seen, heldBack, live);
+ }
+
+ ///
+ /// , or " (2)", "(3)"… (within ):
+ /// the first no project of but has.
+ ///
+ internal static string FreeName(IReadOnlyList live, string name, Guid? projectId)
+ {
+ var taken = live.Where(p => p.Id != projectId).Select(p => p.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
+ var candidate = name;
+ for (var n = 2; taken.Contains(candidate); n++)
{
- if (seen.Contains(gone.Id)) continue;
- Log.Warning("projects", $"Project {gone.Id} of sequence {sequenceId} has no folder any more; it is marked deleted");
- await db.Projects.UpdateAsync(gone with { DeletedAt = DateTimeOffset.UtcNow, UpdatedAt = DateTimeOffset.UtcNow }, cancellationToken).ConfigureAwait(false);
+ var suffix = $" ({n.ToString(CultureInfo.InvariantCulture)})";
+ candidate = (name.Length + suffix.Length > ProjectService.MaxNameLength ? name[..(ProjectService.MaxNameLength - suffix.Length)].TrimEnd() : name) + suffix;
}
+ return candidate;
}
/// The project folders of a sequence: the live ones, then those in the projects' trash (<id>-<time> ).
@@ -67,25 +114,26 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
///
/// The record with the version files that follow its head adopted (a push that wrote its version but not the record),
- /// and the files past the head that do not follow it moved aside; the record is written again when its head moved.
+ /// and the files past the head that do not follow it moved aside; the record is written again when its head moved. A
+ /// file past the head that cannot be read proves nothing: then none is moved aside.
///
- private static ProjectRecord RollForward(SequencePaths folder, ProjectRecord record, string directory)
+ private static ProjectRecord RollForward(SequencePaths folder, ProjectRecord record, string directory, bool writable)
{
var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
var files = ProjectSidecars.ListVersions(versions);
var head = record.Head;
var moved = false;
+ var unread = false;
while (true)
{
var next = (head?.Number ?? 0) + 1;
- var candidates = files.Where(f => f.Number == next).ToList();
ProjectVersion? follower = null;
- foreach (var candidate in candidates)
+ foreach (var candidate in files.Where(f => f.Number == next))
{
try
{
var version = ProjectSidecars.ReadVersion(candidate.Path);
- if (version.ProjectId == record.Id && version.ParentId == head?.Id)
+ if (version.ProjectId == record.Id && version.Id == candidate.Id && version.ParentId == head?.Id)
{
follower = version;
break;
@@ -94,86 +142,192 @@ internal sealed class ProjectIndexer(ChronosDatabase db)
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
Log.Warning("projects", $"Version file {candidate.Path} cannot be read", ex);
+ unread = true;
}
}
if (follower is null) break;
head = new ProjectHead(follower.Id, follower.Number);
moved = true;
}
- foreach (var stray in files.Where(f => f.Number > (head?.Number ?? 0)))
+ if (writable && !unread)
{
- try
- {
- var aside = Path.Combine(versions, StrayFolder);
- Directory.CreateDirectory(aside);
- File.Move(stray.Path, Path.Combine(aside, Path.GetFileName(stray.Path) + "." + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff", CultureInfo.InvariantCulture)));
- Log.Warning("projects", $"Version file {stray.Path} does not follow the head of project {record.Id}; moved aside");
- }
- catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
- {
- Log.Warning("projects", $"Version file {stray.Path} could not be moved aside", ex);
- }
+ foreach (var stray in files.Where(f => f.Number > (head?.Number ?? 0)))
+ SetAside(versions, stray.Path, $"does not follow the head of project {record.Id}");
}
if (!moved) return record;
var rolled = record with { Head = head, Updated = DateTimeOffset.UtcNow };
- ProjectSidecars.WriteRecord(folder.ProjectRecordPath(record.Id), rolled);
+ if (!writable) return rolled;
+ try
+ {
+ ProjectSidecars.WriteRecord(folder.ProjectRecordPath(record.Id), rolled);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ // The index takes the versions all the same; the record is written with them before anything else is (ProjectService).
+ Log.Warning("projects", $"The record of project {record.Id} could not be moved forward to version {head!.Number}", ex);
+ }
return rolled;
}
- /// Writes what the index lacks of one project's record and versions.
- private async Task IndexAsync(ProjectRecord record, string directory, CancellationToken cancellationToken)
+ /// A version file moved aside into versions/.stray (kept, never deleted).
+ private static void SetAside(string versions, string path, string why)
+ {
+ try
+ {
+ var aside = Path.Combine(versions, StrayFolder);
+ Directory.CreateDirectory(aside);
+ File.Move(path, Path.Combine(aside, Path.GetFileName(path) + "." + DateTime.UtcNow.ToString("yyyyMMddHHmmssfff", CultureInfo.InvariantCulture)));
+ Log.Warning("projects", $"Version file {path} {why}; moved aside");
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ Log.Warning("projects", $"Version file {path} could not be moved aside", ex);
+ }
+ }
+
+ ///
+ /// Writes what the index lacks of one project's record and versions. False when its head could not be reached: the
+ /// record names a version this Chronos cannot read (damaged, of a newer Chronos), and the index's head stays on the
+ /// newest one it holds below it.
+ ///
+ private async Task IndexAsync(ProjectRecord record, string directory, bool trashed, bool writable, CancellationToken cancellationToken)
{
var row = await db.Projects.GetAsync(record.Id, cancellationToken).ConfigureAwait(false);
if (row is null)
{
var project = record.ToProject();
- var live = await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
- if (project.DeletedAt is null && project.IsMain && live.Any(p => p.IsMain))
+ if (project.DeletedAt is null)
{
- Log.Warning("projects", $"Project {record.Id} says it is Main, but sequence {record.SequenceId} has one; indexed as another project");
- project = project with { IsMain = false };
+ var live = await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
+ if (project.IsMain && live.Any(p => p.IsMain))
+ {
+ Log.Warning("projects", $"Project {record.Id} says it is Main, but sequence {record.SequenceId} has one; indexed as another project");
+ project = project with { IsMain = false };
+ }
+ project = project with { Name = FreeName(live, project.Name, project.Id) };
}
- if (project.DeletedAt is null && live.Any(p => string.Equals(p.Name, project.Name, StringComparison.OrdinalIgnoreCase)))
- project = project with { Name = project.Name + " (" + project.Id.ToString("N")[..6] + ")" };
// The head follows its versions below, so a version the index cannot take never leaves a head without its row.
- await db.Projects.InsertAsync(project with { HeadVersionId = null, HeadNumber = 0 }, cancellationToken).ConfigureAwait(false);
row = project with { HeadVersionId = null, HeadNumber = 0 };
+ await db.Projects.InsertAsync(row, cancellationToken).ConfigureAwait(false);
}
- var indexed = await db.Projects.ListVersionIdsAsync(record.Id, cancellationToken).ConfigureAwait(false);
- var headNumber = row.HeadNumber;
- Guid? headId = row.HeadVersionId;
- var limit = record.Head?.Number ?? 0;
- foreach (var file in ProjectSidecars.ListVersions(Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder)))
+ await IndexVersionsAsync(record, directory, writable, cancellationToken).ConfigureAwait(false);
+ var (headId, headNumber) = (row.HeadVersionId, row.HeadNumber);
+ var reached = true;
+ if (record.Head is { } head && (head.Number > headNumber || (head.Number == headNumber && head.Id != headId)))
{
- if (file.Number > limit || indexed.ContainsKey(file.Number)) continue;
- ProjectVersion version;
+ if (await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is { } indexed && indexed.ProjectId == record.Id
+ && indexed.Number == head.Number)
+ {
+ (headId, headNumber) = (head.Id, head.Number);
+ }
+ else
+ {
+ reached = false;
+ Log.Warning("projects", $"The newest version of project {record.Id} ({head.Number}) cannot be read; the project is shown as far as it can be, and not changed");
+ var below = await db.Projects.ListVersionsAsync(record.Id, head.Number, 1, cancellationToken).ConfigureAwait(false);
+ if (below.Count > 0 && below[0].Number > headNumber) (headId, headNumber) = (below[0].Id, below[0].Number);
+ }
+ }
+ var updated = row with { HeadVersionId = headId, HeadNumber = headNumber };
+ if (record.Deleted is not null || trashed)
+ {
+ updated = updated with { DeletedAt = row.DeletedAt ?? record.Deleted };
+ }
+ else if (row.DeletedAt is not null || row.Name != record.Name)
+ {
+ var others = (await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
+ .Where(p => p.Id != record.Id).ToList();
+ if (row.DeletedAt is not null)
+ {
+ // Its live record says it is not deleted: listed again, Main again unless the sequence has another.
+ Log.Warning("projects", $"Project {record.Id} of sequence {record.SequenceId} is live in its sidecars; it is listed again");
+ updated = updated with { DeletedAt = null, IsMain = row.IsMain && !others.Any(p => p.IsMain) };
+ }
+ // The record's name, else (another project has it) the name the index has, else one free.
+ updated = updated with
+ {
+ Name = FreeName(others, record.Name, record.Id) == record.Name ? record.Name
+ : FreeName(others, row.Name, record.Id) == row.Name ? row.Name
+ : FreeName(others, record.Name, record.Id),
+ };
+ }
+ if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false);
+ return reached || record.Deleted is not null || trashed;
+ }
+
+ ///
+ /// The version files up to the record's head the index lacks, taken in. Of several files of one number, the one the
+ /// record's chain names is taken and the others are moved aside; when the chain cannot tell (a version on it cannot be
+ /// read), none is taken.
+ ///
+ private async Task IndexVersionsAsync(ProjectRecord record, string directory, bool writable, CancellationToken cancellationToken)
+ {
+ var versions = Path.Combine(directory, ArchiveLayout.ProjectVersionsFolder);
+ var files = ProjectSidecars.ListVersions(versions);
+ var limit = record.Head?.Number ?? 0;
+ var indexed = new Dictionary(await db.Projects.ListVersionIdsAsync(record.Id, cancellationToken).ConfigureAwait(false));
+ Dictionary? chain = null;
+ foreach (var number in files.Where(f => f.Number <= limit && !indexed.ContainsKey(f.Number)).GroupBy(f => f.Number))
+ {
+ var candidates = number.ToList();
+ if (candidates.Count > 1)
+ {
+ chain ??= Chain(record, files);
+ var named = candidates.Where(c => chain.TryGetValue(c.Number, out var id) && id == c.Id).ToList();
+ if (named.Count == 0)
+ {
+ Log.Warning("projects", $"Project {record.Id} has {candidates.Count} version files numbered {number.Key} and its record's chain names none; none is indexed");
+ continue;
+ }
+ if (writable)
+ {
+ foreach (var other in candidates.Where(c => c.Id != named[0].Id))
+ SetAside(versions, other.Path, $"has the number of another version of project {record.Id}");
+ }
+ candidates = named;
+ }
+ foreach (var file in candidates)
+ {
+ ProjectVersion version;
+ try
+ {
+ version = ProjectSidecars.ReadVersion(file.Path);
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ Log.Warning("projects", $"Version file {file.Path} cannot be read; it is not indexed", ex);
+ continue;
+ }
+ if (version.ProjectId != record.Id || version.Number != file.Number || version.Id != file.Id) continue;
+ if (await db.Projects.GetVersionAsync(version.Id, cancellationToken).ConfigureAwait(false) is not null)
+ {
+ Log.Warning("projects", $"Version file {file.Path} holds a version the index has under another number; it is not indexed");
+ continue;
+ }
+ await db.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
+ indexed[version.Number] = version.Id;
+ }
+ }
+ }
+
+ /// The version ids by number along the record's chain, from its head back through each parent, as far as their files read.
+ private static Dictionary Chain(ProjectRecord record, IReadOnlyList<(int Number, Guid Id, string Path)> files)
+ {
+ var chain = new Dictionary();
+ var byId = files.GroupBy(f => f.Id).ToDictionary(g => g.Key, g => g.First());
+ for (var id = record.Head?.Id; id is { } current && byId.TryGetValue(current, out var file) && !chain.ContainsKey(file.Number);)
+ {
+ chain[file.Number] = file.Id;
try
{
- version = ProjectSidecars.ReadVersion(file.Path);
+ id = ProjectSidecars.ReadVersion(file.Path).ParentId;
}
catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
{
- Log.Warning("projects", $"Version file {file.Path} cannot be read; it is not indexed", ex);
- continue;
+ Log.Warning("projects", $"Version file {file.Path} cannot be read", ex);
+ break;
}
- if (version.ProjectId != record.Id || version.Number != file.Number) continue;
- await db.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
}
- if (record.Head is { } head && head.Number > headNumber && await db.Projects.GetVersionAsync(head.Id, cancellationToken).ConfigureAwait(false) is not null)
- (headId, headNumber) = (head.Id, head.Number);
- var updated = row with
- {
- Name = row.DeletedAt is null && record.Deleted is null && row.Name != record.Name && !await NameTakenAsync(record, cancellationToken).ConfigureAwait(false)
- ? record.Name
- : row.Name,
- HeadVersionId = headId,
- HeadNumber = headNumber,
- DeletedAt = row.DeletedAt ?? record.Deleted,
- };
- if (updated != row) await db.Projects.UpdateAsync(updated, cancellationToken).ConfigureAwait(false);
+ return chain;
}
-
- private async Task NameTakenAsync(ProjectRecord record, CancellationToken cancellationToken) =>
- (await db.Projects.ListBySequenceAsync(record.SequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false))
- .Any(p => p.Id != record.Id && string.Equals(p.Name, record.Name, StringComparison.OrdinalIgnoreCase));
}
diff --git a/src/Chronos.Data/Projects/ProjectService.cs b/src/Chronos.Data/Projects/ProjectService.cs
index e04f257d..cc925445 100644
--- a/src/Chronos.Data/Projects/ProjectService.cs
+++ b/src/Chronos.Data/Projects/ProjectService.cs
@@ -35,6 +35,13 @@ public sealed class ProjectException(ProjectError reason, string message) : Inva
public ProjectError Reason { get; } = reason;
}
+///
+/// What an operation needs of a sequence's projects cannot be read now: a record damaged, of a newer Chronos or not
+/// readable for a moment, a head whose version this Chronos cannot read, a record ahead of the index. Nothing was written;
+/// reads answer from the index meanwhile. Try again later (or with a newer Chronos).
+///
+public sealed class ProjectsUnreadableException(string message) : IOException(message);
+
/// What became of a version pushed to its home ( ).
public enum PushOutcome
{
@@ -53,8 +60,14 @@ public enum PushOutcome
/// The settings are of a newer format than this home reads: nothing was stored.
NewerFormat,
- /// Not a version of this project: settings that are not portable (they name frames by position), or an id another project holds.
+ ///
+ /// Not a version of this project: settings that are not portable (they name frames by position) or that this home cannot
+ /// read, or an id another project holds.
+ ///
Invalid,
+
+ /// The version's file would be larger than a version may be ( ): nothing was stored.
+ TooLarge,
}
/// The answer to a push: the outcome, the version as stored ( , ) and the project's head.
@@ -76,6 +89,14 @@ public sealed record HomeFrames(int FramesVersion, IReadOnlyList Frames,
/// project.json and settings (positions placed on its frames, through the settings version check), so the code
/// paths that read the sequence's settings keep working and only Main drives the hero frame the Library shows. Other
/// projects stay on this home: copies of a sequence linked to a server sync only its settings, i.e. Main.
+///
+/// The sidecars are the truth. A project the index holds is never taken for gone because its folder is missing or its
+/// record cannot be read: a record that cannot be read holds its project back (listed as the index has it, nothing written
+/// for it, no second Main made beside it), and a project whose folder is missing has its sidecars written again from the
+/// index. A head of a newer settings format is listed and pulled (a workspace opens it read-only) but never mirrored nor
+/// built on here. A read-only archive is read without writing to it: its projects are listed and pulled from its sidecars,
+/// and a sequence with no Main yet gets one in the index only, whose sidecars are written once the archive takes writes.
+///
///
public sealed class ProjectService
{
@@ -87,6 +108,12 @@ public sealed class ProjectService
/// Frames read at once when a sequence's photos are gathered.
private const int PageSize = 1000;
+ /// Versions read at once when a project's sidecars are written again from the index.
+ private const int WriteBackPage = 500;
+
+ /// How far back from Main's head an index that knows no mirror looks for the version its sequence's settings hold.
+ private const int MirrorSearch = 100;
+
/// How long a read waits for a sequence another writer holds before it answers from the index alone.
private static readonly TimeSpan ReadWait = TimeSpan.FromSeconds(3);
@@ -104,6 +131,9 @@ public sealed class ProjectService
/// The client name versions made here carry (the archive engine's lease owner).
public string ClientName => _writer.Options.Owner;
+ /// Whether an archive takes writes now ( ; tests stand in a read-only one).
+ internal Func Availability { get; set; } = ArchiveVolume.Probe;
+
/// The live projects of a sequence, Main first; read from the index alone while the archive is offline.
/// The sequence is not in the index.
public async Task> ListAsync(Guid sequenceId, CancellationToken cancellationToken = default)
@@ -112,16 +142,18 @@ public sealed class ProjectService
return await _db.Projects.ListBySequenceAsync(sequenceId, includeDeleted: false, cancellationToken).ConfigureAwait(false);
}
- /// The sequence's Main project, made now (from its settings) when it has none.
+ /// The sequence's Main project, made now (from its settings) when it has none; in the index only while the archive is read-only.
/// The sequence is not in the index.
/// The archive is offline and the index has no Main project yet.
+ /// The index has no Main and a project record cannot be read now (it may be Main's).
public async Task MainAsync(Guid sequenceId, CancellationToken cancellationToken = default)
{
await ReadAsync(sequenceId, cancellationToken).ConfigureAwait(false);
if (await _db.Projects.GetMainAsync(sequenceId, cancellationToken).ConfigureAwait(false) is { } main) return main;
// The catch-up could not run (the archive offline, the sequence busy): it runs now, or says why it cannot.
- return await UnderLeaseAsync(sequenceId, async (_, ct) => (await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false))!, cancellationToken)
- .ConfigureAwait(false);
+ return await UnderLeaseAsync(sequenceId, async (_, ct) => await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false)
+ ?? throw new ProjectsUnreadableException($"The Main of sequence {sequenceId} cannot be read now."),
+ cancellationToken).ConfigureAwait(false);
}
public Task GetAsync(Guid projectId, CancellationToken cancellationToken = default) => _db.Projects.GetAsync(projectId, cancellationToken);
@@ -149,7 +181,8 @@ public sealed class ProjectService
else
{
var main = (await _db.Projects.GetMainAsync(sequenceId, ct).ConfigureAwait(false))!;
- source = (await _db.Projects.GetVersionAsync(main.HeadVersionId!.Value, ct).ConfigureAwait(false))!;
+ source = await _db.Projects.GetVersionAsync(main.HeadVersionId ?? Guid.Empty, ct).ConfigureAwait(false)
+ ?? throw new ProjectsUnreadableException($"Main of sequence {sequenceId} has no version this Chronos can read.");
}
var now = DateTimeOffset.UtcNow;
var project = new SequenceProject
@@ -176,7 +209,7 @@ public sealed class ProjectService
CreatedAt = now,
StoredAt = now,
};
- return await AddProjectAsync(context.Folder, project, first, ct).ConfigureAwait(false);
+ return await AddProjectAsync(context.Folder, project, first, null, ct).ConfigureAwait(false);
}, cancellationToken);
}
@@ -191,16 +224,18 @@ public sealed class ProjectService
var current = await LiveAsync(projectId, ct).ConfigureAwait(false);
if (current.Name == trimmed) return current;
await EnsureNameFreeAsync(current.SequenceId, projectId, trimmed, ct).ConfigureAwait(false);
+ var record = OwnRecord(context, projectId);
var renamed = current with { Name = trimmed, UpdatedAt = DateTimeOffset.UtcNow };
- ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(renamed));
+ ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(renamed, record.Mirrored));
await _db.Projects.UpdateAsync(renamed, ct).ConfigureAwait(false);
return renamed;
}, cancellationToken).ConfigureAwait(false);
}
///
- /// Deletes a project: its record says so, its folder moves to .chronos/projects/.trash (kept, never erased by
- /// Chronos) and it leaves the listing.
+ /// Deletes a project: its folder moves to .chronos/projects/.trash (kept, never erased by Chronos), then its
+ /// record there says so, and it leaves the listing. A folder that cannot be moved (a file in it held open) fails the
+ /// delete with nothing changed.
///
/// Unknown project, or the Main project.
public async Task DeleteAsync(Guid projectId, CancellationToken cancellationToken = default)
@@ -210,12 +245,22 @@ public sealed class ProjectService
await UnderLeaseAsync(project.SequenceId, async (context, ct) =>
{
var current = await LiveAsync(projectId, ct).ConfigureAwait(false);
+ var record = OwnRecord(context, projectId);
var now = DateTimeOffset.UtcNow;
var deleted = current with { DeletedAt = now, UpdatedAt = now };
- ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(projectId), ProjectRecord.Of(deleted));
Directory.CreateDirectory(context.Folder.ProjectsTrashDirectory);
- Directory.Move(context.Folder.ProjectDirectory(projectId),
- Path.Combine(context.Folder.ProjectsTrashDirectory, projectId.ToString("N") + "-" + now.UtcTicks.ToString(System.Globalization.CultureInfo.InvariantCulture)));
+ var trashed = Path.Combine(context.Folder.ProjectsTrashDirectory,
+ projectId.ToString("N") + "-" + now.UtcTicks.ToString(System.Globalization.CultureInfo.InvariantCulture));
+ Directory.Move(context.Folder.ProjectDirectory(projectId), trashed);
+ try
+ {
+ ProjectSidecars.WriteRecord(Path.Combine(trashed, ArchiveLayout.ProjectFile), ProjectRecord.Of(deleted, record.Mirrored));
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ // In the projects' trash it is deleted all the same (ProjectIndexer): the record only says when.
+ Log.Warning("projects", $"The record of deleted project {projectId} could not say so", ex);
+ }
await _db.Projects.UpdateAsync(deleted, ct).ConfigureAwait(false);
return true;
}, cancellationToken).ConfigureAwait(false);
@@ -275,10 +320,13 @@ public sealed class ProjectService
? new PushResult(PushOutcome.AlreadyStored, existing, head)
: new PushResult(PushOutcome.Invalid, null, head);
}
+ // A project whose record or head cannot be read now takes nothing: its history may go on past what the index holds.
+ if (context.Reconcile.HeldBack.Contains(projectId))
+ throw new ProjectsUnreadableException($"Project {projectId} cannot be read now; the version is kept to be sent again.");
if (!ProjectContent.Verifies(version.SettingsJson, version.ContentHash)) return new PushResult(PushOutcome.HashMismatch, null, head);
var format = ProjectContent.FormatOf(version.SettingsJson);
if (format > ProjectSettings.CurrentVersion) return new PushResult(PushOutcome.NewerFormat, null, head);
- if (!IsPortable(version.SettingsJson)) return new PushResult(PushOutcome.Invalid, null, head);
+ if (!IsPortable(version.SettingsJson) || !Readable(version.SettingsJson)) return new PushResult(PushOutcome.Invalid, null, head);
if (version.ParentId != current.HeadVersionId) return new PushResult(PushOutcome.Diverged, null, head);
var stored = version with
{
@@ -288,8 +336,21 @@ public sealed class ProjectService
SettingsFormat = format,
StoredAt = DateTimeOffset.UtcNow,
};
+ // Never stored when it could not be read back (the same limit for both).
+ if (!ProjectSidecars.Fits(stored)) return new PushResult(PushOutcome.TooLarge, null, head);
current = await AppendAsync(context.Folder, current, stored, ct).ConfigureAwait(false);
- if (current.IsMain) await MirrorAsync(context, current, stored, ct).ConfigureAwait(false);
+ if (current.IsMain)
+ {
+ try
+ {
+ await MirrorAsync(context, current, stored, ct).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
+ {
+ // The version is stored: the next operation's catch-up completes the mirror.
+ Log.Warning("projects", $"Version {stored.Number} of Main of sequence {current.SequenceId} is stored but not mirrored yet", ex);
+ }
+ }
return new PushResult(PushOutcome.Stored, stored, stored);
}, cancellationToken).ConfigureAwait(false);
}
@@ -307,9 +368,42 @@ public sealed class ProjectService
}
}
+ ///
+ /// Whether this archive holds of the project durably: its file, read back now, is that version
+ /// with that content hash (the index's row alone proves nothing: a file damaged or taken away since is not seen there).
+ /// False when the file is missing, damaged, of a newer Chronos or another version's, or the project was deleted: what a
+ /// workspace checks before it drops its own copy of a version.
+ ///
+ /// The archive is offline.
+ /// The sequence's folder does not hold it now (another index of the archive moved it).
+ public async Task StoredAsync(Guid projectId, ProjectVersion version, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(version);
+ if (await _db.Projects.GetAsync(projectId, cancellationToken).ConfigureAwait(false) is not { DeletedAt: null } project) return false;
+ var (paths, row) = await LocateAsync(project.SequenceId, cancellationToken).ConfigureAwait(false);
+ if (Availability(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
+ var folder = paths.Sequence(row.Folder);
+ if (!SequenceFolders.Holds(folder, project.SequenceId)) throw new SequenceFolderMovedException();
+ var number = (await _db.Projects.GetVersionAsync(version.Id, cancellationToken).ConfigureAwait(false))?.Number ?? version.Number;
+ var path = folder.VersionPath(projectId, number, version.Id);
+ try
+ {
+ var stored = ProjectSidecars.ReadVersion(path);
+ if (stored.Id == version.Id && stored.ProjectId == projectId && string.Equals(stored.ContentHash, version.ContentHash, StringComparison.OrdinalIgnoreCase))
+ return true;
+ Log.Warning("projects", $"{path} does not hold version {version.Id} as it was sent");
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ Log.Warning("projects", $"Version {number} of project {projectId} cannot be read back from the archive", ex);
+ }
+ return false;
+ }
+
///
/// Catches the sequence's projects up under its lease (see the class remarks) before a read; the index alone answers while
- /// the archive is offline or the sequence busy for longer than the lease wait.
+ /// the archive is offline, the sequence busy for longer than the lease wait, or a project of it cannot be read now. A
+ /// read-only archive is caught up without writing to it ( ).
///
private async Task ReadAsync(Guid sequenceId, CancellationToken cancellationToken)
{
@@ -317,7 +411,38 @@ public sealed class ProjectService
{
await UnderLeaseAsync(sequenceId, (_, _) => Task.FromResult(true), cancellationToken, ReadWait).ConfigureAwait(false);
}
- catch (Exception ex) when (ex is ArchiveOfflineException or LeaseUnavailableException or SequenceFolderMovedException)
+ catch (ArchiveOfflineException ex) when (ex.Availability == ArchiveAvailability.ReadOnly)
+ {
+ await ReadOnlyCatchUpAsync(sequenceId, cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is ArchiveOfflineException or LeaseUnavailableException or SequenceFolderMovedException or ProjectsUnreadableException)
+ {
+ Log.Info("projects", $"The projects of sequence {sequenceId} are read from the index: {ex.Message}");
+ }
+ }
+
+ ///
+ /// The catch-up of a sequence of a read-only archive (a backup mounted read-only, a share this PC may only read): its
+ /// sidecars read into the index as they are and nothing written to the archive (no lease, no record moved forward, no
+ /// version set aside, no mirror). A sequence with no Main anywhere gets one in the index only, from its settings, so it
+ /// opens; its sidecars are written once the archive takes writes ( ).
+ ///
+ private async Task ReadOnlyCatchUpAsync(Guid sequenceId, CancellationToken cancellationToken)
+ {
+ var (paths, row) = await LocateAsync(sequenceId, cancellationToken).ConfigureAwait(false);
+ var folder = paths.Sequence(row.Folder);
+ try
+ {
+ if (!SequenceFolders.Holds(folder, sequenceId))
+ {
+ Log.Info("projects", $"The folder of sequence {sequenceId} does not hold it now; its projects are read from the index");
+ return;
+ }
+ var context = new LeaseContext(folder, row, await _indexer.ReconcileAsync(folder, sequenceId, cancellationToken, writable: false).ConfigureAwait(false));
+ if (context.Reconcile.Complete && !await HasMainAsync(context, cancellationToken).ConfigureAwait(false))
+ await MakeMainAsync(context, writable: false, cancellationToken).ConfigureAwait(false);
+ }
+ catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Log.Info("projects", $"The projects of sequence {sequenceId} are read from the index: {ex.Message}");
}
@@ -334,34 +459,66 @@ public sealed class ProjectService
private async Task UnderLeaseAsync(Guid sequenceId, Func> work, CancellationToken cancellationToken,
TimeSpan? wait = null)
{
- var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
- var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
- var paths = new ArchivePaths(archive.RootPath);
- ArchiveVolume.EnsureWritable(paths);
+ var (paths, _) = await LocateAsync(sequenceId, cancellationToken).ConfigureAwait(false);
+ var availability = Availability(paths);
+ if (availability != ArchiveAvailability.Online) throw new ArchiveOfflineException(paths.Root, availability);
await using (await _writer.LockAsync(paths, sequenceId, wait, cancellationToken).ConfigureAwait(false))
{
var row = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
var folder = paths.Sequence(row.Folder);
if (!SequenceFolders.Holds(folder, sequenceId)) throw new SequenceFolderMovedException();
- var context = new LeaseContext(folder, row);
+ var context = new LeaseContext(folder, row, await _indexer.ReconcileAsync(folder, sequenceId, cancellationToken).ConfigureAwait(false));
await CatchUpAsync(context, cancellationToken).ConfigureAwait(false);
return await work(context, cancellationToken).ConfigureAwait(false);
}
}
- /// The sidecars into the index, the Main project made, and Main in step with the sequence's settings.
+ /// The archive of a sequence and its index row.
+ private async Task<(ArchivePaths Paths, Sequence Row)> LocateAsync(Guid sequenceId, CancellationToken cancellationToken)
+ {
+ var sequence = await _db.Sequences.GetAsync(sequenceId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
+ var archive = await _db.Archives.GetAsync(sequence.ArchiveId, cancellationToken).ConfigureAwait(false) ?? throw SequenceMissing(sequenceId);
+ return (new ArchivePaths(archive.RootPath), sequence);
+ }
+
+ ///
+ /// After the sidecars came into the index ( ): the projects the archive has no folder for
+ /// written back, the Main project made when the sequence has none, and Main in step with the sequence's settings.
+ ///
private async Task CatchUpAsync(LeaseContext context, CancellationToken cancellationToken)
{
- await _indexer.ReconcileAsync(context.Folder, context.Row.Id, cancellationToken).ConfigureAwait(false);
+ await WriteBackAsync(context, cancellationToken).ConfigureAwait(false);
var main = await _db.Projects.GetMainAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
- if (main?.HeadVersionId is null)
+ if (main is null)
{
- await MakeMainAsync(context, main, cancellationToken).ConfigureAwait(false);
+ // A record that cannot be read now may be Main's: no second Main is made beside it.
+ if (!context.Reconcile.Complete)
+ throw new ProjectsUnreadableException($"Sequence {context.Row.Id} has no Main in the index and a project of it cannot be read now.");
+ if (!await HasMainAsync(context, cancellationToken).ConfigureAwait(false))
+ await MakeMainAsync(context, writable: true, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ if (context.Reconcile.HeldBack.Contains(main.Id) || main.HeadVersionId is not { } headId)
+ {
+ // Nothing is mirrored from, merged into or built on a Main whose record or head cannot be read now.
+ Log.Info("projects", $"Main of sequence {context.Row.Id} cannot be read now; it is caught up once it can");
+ return;
+ }
+ if (main.MirroredVersionId == headId && main.MirroredSettingsVersion == context.Row.SettingsVersion) return;
+ var head = (await _db.Projects.GetVersionAsync(headId, cancellationToken).ConfigureAwait(false))!;
+ if (IsNewer(head))
+ {
+ // A head of a newer Chronos (another PC sharing the archive): listed and pulled (a workspace opens it read-only), never
+ // mirrored, merged into or built on here; the sequence's settings stay as that Chronos wrote them.
+ Log.Info("projects", $"Main of sequence {context.Row.Id} is at version {head.Number} of a newer Chronos; the sequence's settings are left to it");
return;
}
- if (main.MirroredVersionId == main.HeadVersionId && main.MirroredSettingsVersion == context.Row.SettingsVersion) return;
- var head = (await _db.Projects.GetVersionAsync(main.HeadVersionId.Value, cancellationToken).ConfigureAwait(false))!;
var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
+ if (main.MirroredVersionId is null)
+ {
+ await MeetMainAsync(context, main, head, frames.Photos, cancellationToken).ConfigureAwait(false);
+ return;
+ }
if (CurrentPortable(context, frames.Photos) is not { } current)
{
// Settings no Chronos can read (or of a newer one): left as they are, recorded as seen.
@@ -370,10 +527,10 @@ public sealed class ProjectService
}
// The head parks its photos as the frames held them when it was made: a conversion or a hash since names them anew.
var sameAsHead = ProjectContent.SameChoices(head.SettingsJson, current, frames.Photos);
- if (main.MirroredVersionId == main.HeadVersionId || main.MirroredVersionId is null)
+ if (main.MirroredVersionId == main.HeadVersionId)
{
- // Only the sequence's settings version moved (settings another path wrote, or only its title or tags), or the index
- // was rebuilt and knows no mirror: settings that differ from the head are its newest state.
+ // Only the sequence's settings version moved (settings another path wrote, or only its title or tags): settings that
+ // differ from the head are its newest state.
var latest = sameAsHead ? head : await AppendDerivedAsync(context, main, head, current, ProjectVersionKind.External, cancellationToken).ConfigureAwait(false);
await _db.Projects.SetMirroredAsync(main.Id, latest.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
return;
@@ -401,13 +558,145 @@ public sealed class ProjectService
await MirrorAsync(context, main, version, cancellationToken).ConfigureAwait(false);
}
- /// The Main project of a sequence that has none (or one without versions): version 1 from the sequence's settings.
- private async Task MakeMainAsync(LeaseContext context, SequenceProject? existing, CancellationToken cancellationToken)
+ ///
+ /// Main as this index meets it with no mirror known (an index rebuilt from the sidecars, another PC's index of the archive,
+ /// an index taken back from a backup): the sequence's settings are what its project.json holds, the file every
+ /// mirror writes, never this index's copy, which may have been read long ago. The head's choices: in step, this index's
+ /// copy brought up to them. Those of the version Main's record says was mirrored last (an older record names none: of a
+ /// version the head descends from): a mirror a crash cut short, completed. Neither: settings another path wrote since,
+ /// an external version.
+ ///
+ private async Task MeetMainAsync(LeaseContext context, SequenceProject main, ProjectVersion head, FramePhotos photos, CancellationToken cancellationToken)
+ {
+ if (SharedPortable(context, photos) is not { } shared)
+ {
+ // Settings no Chronos can read (or of a newer one): left as they are, recorded as seen.
+ await _db.Projects.SetMirroredAsync(main.Id, head.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ if (ProjectContent.SameChoices(head.SettingsJson, shared, photos))
+ {
+ if (CurrentPortable(context, photos) is { } copy && ProjectContent.SameChoices(head.SettingsJson, copy, photos))
+ await _db.Projects.SetMirroredAsync(main.Id, head.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
+ else
+ await MirrorAsync(context, main, head, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ if (await MirroredBeforeAsync(context, main, head, shared, photos, cancellationToken).ConfigureAwait(false))
+ {
+ await MirrorAsync(context, main, head, cancellationToken).ConfigureAwait(false);
+ return;
+ }
+ var external = await AppendDerivedAsync(context, main, head, shared, ProjectVersionKind.External, cancellationToken).ConfigureAwait(false);
+ main = (await _db.Projects.GetAsync(main.Id, cancellationToken).ConfigureAwait(false))!;
+ await MirrorAsync(context, main, external, cancellationToken).ConfigureAwait(false);
+ }
+
+ ///
+ /// Whether holds the choices of an earlier version of Main a mirror wrote: the one Main's record
+ /// names as mirrored last, else (an older record) one of the versions the head descends from.
+ ///
+ private async Task MirroredBeforeAsync(LeaseContext context, SequenceProject main, ProjectVersion head, string shared, FramePhotos photos,
+ CancellationToken cancellationToken)
+ {
+ if (context.Reconcile.Live.TryGetValue(main.Id, out var record) && record.Mirrored is { } recorded)
+ {
+ return recorded.Id != head.Id && await _db.Projects.GetVersionAsync(recorded.Id, cancellationToken).ConfigureAwait(false) is { } mirrored
+ && mirrored.ProjectId == main.Id && !IsNewer(mirrored) && ProjectContent.SameChoices(mirrored.SettingsJson, shared, photos);
+ }
+ var parent = head.ParentId;
+ for (var step = 0; parent is { } id && step < MirrorSearch; step++)
+ {
+ if (await _db.Projects.GetVersionAsync(id, cancellationToken).ConfigureAwait(false) is not { } version || version.ProjectId != main.Id) break;
+ if (!IsNewer(version) && ProjectContent.SameChoices(version.SettingsJson, shared, photos)) return true;
+ parent = version.ParentId;
+ }
+ return false;
+ }
+
+ ///
+ /// Whether the sequence has a Main the archive does not say is gone: a live row, or one this index marked deleted (an
+ /// older Chronos took a record it could not read for gone) whose folder the archive does not have.
+ ///
+ private async Task HasMainAsync(LeaseContext context, CancellationToken cancellationToken) =>
+ (await _db.Projects.ListBySequenceAsync(context.Row.Id, includeDeleted: true, cancellationToken).ConfigureAwait(false))
+ .Any(p => p.IsMain && (p.DeletedAt is null || !context.Reconcile.Seen.Contains(p.Id)));
+
+ ///
+ /// The projects the index holds that the archive has no folder for (a projects folder moved away and put back without
+ /// them, a sync tool, a Main made while the archive was read-only), with a Main this index marked deleted only because an
+ /// older Chronos took its unreadable record for gone: their sidecars written again from the index, which holds every
+ /// version, so the archive is their truth again and nothing is lost. Never over a folder that is there, nor into a
+ /// sequence folder without its sidecar. A Main beside which the archive has another live Main record is written back as
+ /// another project (the one in the archive stays Main). A project that cannot be written is tried again next time.
+ ///
+ private async Task WriteBackAsync(LeaseContext context, CancellationToken cancellationToken)
+ {
+ if (!File.Exists(context.Folder.ManifestPath)) return;
+ var rows = await _db.Projects.ListBySequenceAsync(context.Row.Id, includeDeleted: true, cancellationToken).ConfigureAwait(false);
+ var liveMain = rows.Any(p => p.IsMain && p.DeletedAt is null);
+ foreach (var row in rows.Where(p => !context.Reconcile.Seen.Contains(p.Id) && (p.DeletedAt is null || (p.IsMain && !liveMain))))
+ {
+ if (Directory.Exists(context.Folder.ProjectDirectory(row.Id))) continue;
+ var others = (await _db.Projects.ListBySequenceAsync(context.Row.Id, includeDeleted: false, cancellationToken).ConfigureAwait(false))
+ .Where(p => p.Id != row.Id).ToList();
+ var project = row with { DeletedAt = null, Name = ProjectIndexer.FreeName(others, row.Name, row.Id) };
+ SequenceProject? promoted = null;
+ if (project.IsMain && context.Reconcile.Live.Values.FirstOrDefault(r => r.IsMain && r.Id != row.Id) is { } theirs
+ && others.FirstOrDefault(p => p.Id == theirs.Id) is { } other)
+ {
+ // Another index made the sequence's Main meanwhile, into the archive: that one is Main, this one another project.
+ var rest = others.Where(p => p.Id != other.Id).ToList();
+ promoted = other with { IsMain = true, Name = ProjectIndexer.FreeName(rest, theirs.Name, other.Id) };
+ project = project with
+ {
+ IsMain = false,
+ Name = ProjectIndexer.FreeName([.. rest, promoted], row.Name, row.Id),
+ MirroredVersionId = null,
+ MirroredSettingsVersion = null,
+ };
+ }
+ try
+ {
+ for (var after = 0; ;)
+ {
+ var page = await _db.Projects.ListVersionsAfterAsync(row.Id, after, WriteBackPage, cancellationToken).ConfigureAwait(false);
+ foreach (var version in page) ProjectSidecars.WriteVersion(context.Folder.VersionPath(row.Id, version.Number, version.Id), version);
+ if (page.Count < WriteBackPage) break;
+ after = page[^1].Number;
+ }
+ var mirrored = project.IsMain && project.MirroredVersionId is { } mirroredId
+ && await _db.Projects.GetVersionAsync(mirroredId, cancellationToken).ConfigureAwait(false) is { } known
+ ? new ProjectHead(known.Id, known.Number)
+ : null;
+ ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(row.Id), ProjectRecord.Of(project, mirrored));
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ Log.Warning("projects", $"The sidecars of project {row.Id} could not be written again from the index; tried again next time", ex);
+ continue;
+ }
+ Log.Warning("projects", $"Project {row.Id} of sequence {context.Row.Id} had no folder in the archive; its sidecars are written again from the index");
+ if (project == row) continue;
+ await _db.Projects.UpdateAsync(project, cancellationToken).ConfigureAwait(false);
+ if (promoted is not null) await _db.Projects.UpdateAsync(promoted, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ ///
+ /// The Main project of a sequence that has none: version 1 from the sequence's settings as its project.json holds
+ /// them (this index's copy when there is none, else defaults), this index's copy brought up to them. With the archive
+ /// read-only it is made in the index only, and nothing is written to the archive: its sidecars are written once the
+ /// archive takes writes ( ).
+ ///
+ private async Task MakeMainAsync(LeaseContext context, bool writable, CancellationToken cancellationToken)
{
var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
- var portable = CurrentPortable(context, frames.Photos) ?? ProjectContent.ToPortable(new ProjectSettings(), frames.Photos);
+ var shared = SharedPortable(context, frames.Photos);
+ var copy = CurrentPortable(context, frames.Photos);
+ var portable = shared ?? copy ?? ProjectContent.ToPortable(new ProjectSettings(), frames.Photos);
var now = DateTimeOffset.UtcNow;
- var project = existing ?? new SequenceProject
+ var project = new SequenceProject
{
Id = Guid.CreateVersion7(),
SequenceId = context.Row.Id,
@@ -416,17 +705,19 @@ public sealed class ProjectService
CreatedAt = now,
UpdatedAt = now,
};
- var first = NewVersion(project.Id, null, project.HeadNumber + 1, ProjectVersionKind.Imported, portable, context.Row.FramesVersion, now);
- if (existing is null)
+ var first = NewVersion(project.Id, null, 1, ProjectVersionKind.Imported, portable, context.Row.FramesVersion, now);
+ // Settings this Chronos cannot read (of a newer one) are never written over: the index's copy then stays as it is too.
+ var inStep = shared is null || copy is null || ProjectContent.SameChoices(portable, copy, frames.Photos);
+ if (inStep) project = project with { MirroredSettingsVersion = context.Row.SettingsVersion, MirroredVersionId = first.Id };
+ if (!writable)
{
- await AddProjectAsync(context.Folder, project with { MirroredSettingsVersion = context.Row.SettingsVersion, MirroredVersionId = first.Id }, first,
- cancellationToken).ConfigureAwait(false);
- }
- else
- {
- var updated = await AppendAsync(context.Folder, existing, first, cancellationToken).ConfigureAwait(false);
- await _db.Projects.SetMirroredAsync(updated.Id, first.Id, context.Row.SettingsVersion, cancellationToken).ConfigureAwait(false);
+ await AddProjectAsync(null, project, first, null, cancellationToken).ConfigureAwait(false);
+ return;
}
+ var onDisk = shared is not null && File.Exists(context.Folder.ProjectPath);
+ var made = await AddProjectAsync(context.Folder, project, first, onDisk ? new ProjectHead(first.Id, first.Number) : null, cancellationToken)
+ .ConfigureAwait(false);
+ if (!inStep) await MirrorAsync(context, made, first, cancellationToken).ConfigureAwait(false);
}
/// "Main", or "Main (2)"… when a project of that name exists already (one made before the sequence had its Main).
@@ -440,10 +731,19 @@ public sealed class ProjectService
}
/// The portable form of the sequence's settings in force (index, else project.json , else defaults); null when unreadable or newer.
- private static string? CurrentPortable(LeaseContext context, FramePhotos photos)
+ private static string? CurrentPortable(LeaseContext context, FramePhotos photos) =>
+ Portable(context, context.Row.SettingsJson ?? (File.Exists(context.Folder.ProjectPath) ? AtomicFile.ReadAllText(context.Folder.ProjectPath) : null), photos);
+
+ ///
+ /// The portable form of the sequence's settings as the archive holds them, in project.json (else this index's copy,
+ /// else defaults); null when unreadable or newer.
+ ///
+ private static string? SharedPortable(LeaseContext context, FramePhotos photos) =>
+ Portable(context, File.Exists(context.Folder.ProjectPath) ? AtomicFile.ReadAllText(context.Folder.ProjectPath) : context.Row.SettingsJson, photos);
+
+ private static string? Portable(LeaseContext context, string? json, FramePhotos photos)
{
- var json = context.Row.SettingsJson ?? (File.Exists(context.Folder.ProjectPath) ? AtomicFile.ReadAllText(context.Folder.ProjectPath) : null)
- ?? new ProjectSettings().ToJson();
+ json ??= new ProjectSettings().ToJson();
try
{
if (ProjectContent.FormatOf(json) > ProjectSettings.CurrentVersion) return null;
@@ -456,6 +756,10 @@ public sealed class ProjectService
}
}
+ /// Whether a version holds settings of a newer Chronos (shown and pulled, never mirrored nor built on here).
+ private static bool IsNewer(ProjectVersion version) =>
+ version.SettingsFormat > ProjectSettings.CurrentVersion || ProjectContent.FormatOf(version.SettingsJson) > ProjectSettings.CurrentVersion;
+
/// A version made here (catch-up or merge) appended to Main.
private async Task AppendDerivedAsync(LeaseContext context, SequenceProject main, ProjectVersion head, string portable, ProjectVersionKind kind,
CancellationToken cancellationToken)
@@ -481,12 +785,19 @@ public sealed class ProjectService
StoredAt = now,
};
- /// A new project with its first version: sidecars (version, then record), then both rows in one transaction.
- private async Task AddProjectAsync(SequencePaths folder, SequenceProject project, ProjectVersion first, CancellationToken cancellationToken)
+ ///
+ /// A new project with its first version: sidecars (version, then the record, naming ), then
+ /// both rows in one transaction; the rows only when is null (a read-only archive).
+ ///
+ private async Task AddProjectAsync(SequencePaths? folder, SequenceProject project, ProjectVersion first, ProjectHead? mirrored,
+ CancellationToken cancellationToken)
{
var withHead = project with { HeadVersionId = first.Id, HeadNumber = first.Number };
- ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, first.Number, first.Id), first);
- ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(withHead));
+ if (folder is not null)
+ {
+ ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, first.Number, first.Id), first);
+ ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(withHead, mirrored));
+ }
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
await tx.Projects.InsertAsync(withHead with { HeadVersionId = null, HeadNumber = 0 }, cancellationToken).ConfigureAwait(false);
await tx.Projects.InsertVersionAsync(first, cancellationToken).ConfigureAwait(false);
@@ -495,13 +806,21 @@ public sealed class ProjectService
return withHead;
}
- /// A version appended to a project: its sidecar, the record's head, then the version row and the head in one transaction.
+ ///
+ /// A version appended to a project: its sidecar, the record's head, then the version row and the head in one
+ /// transaction. Only onto the head the record names, read again now: a record that cannot be read, or that names another
+ /// head than the index has (a version this Chronos cannot read), is never written over and the version is not made.
+ ///
+ /// The record cannot be read, or its head is not the index's.
private async Task AppendAsync(SequencePaths folder, SequenceProject project, ProjectVersion version, CancellationToken cancellationToken)
{
+ var record = ReadOwnRecord(folder, project.Id);
+ if (record.Head != (project.HeadVersionId is { } headId ? new ProjectHead(headId, project.HeadNumber) : null))
+ throw new ProjectsUnreadableException($"The record of project {project.Id} names version {record.Head?.Number} as its head, which the index does not hold.");
var now = DateTimeOffset.UtcNow;
var advanced = project with { HeadVersionId = version.Id, HeadNumber = version.Number, UpdatedAt = now };
ProjectSidecars.WriteVersion(folder.VersionPath(project.Id, version.Number, version.Id), version);
- ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(advanced));
+ ProjectSidecars.WriteRecord(folder.ProjectRecordPath(project.Id), ProjectRecord.Of(advanced, record.Mirrored));
await using var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false);
await tx.Projects.InsertVersionAsync(version, cancellationToken).ConfigureAwait(false);
if (!await tx.Projects.AdvanceHeadAsync(project.Id, project.HeadNumber, version.Id, version.Number, now, cancellationToken).ConfigureAwait(false))
@@ -510,18 +829,58 @@ public sealed class ProjectService
return advanced;
}
+ /// The record of a live project as its folder holds it now, before it is written again: one held back or unreadable is never written over.
+ /// The project is held back, or its record cannot be read.
+ private static ProjectRecord OwnRecord(LeaseContext context, Guid projectId) =>
+ context.Reconcile.HeldBack.Contains(projectId)
+ ? throw new ProjectsUnreadableException($"Project {projectId} cannot be read now.")
+ : ReadOwnRecord(context.Folder, projectId);
+
+ private static ProjectRecord ReadOwnRecord(SequencePaths folder, Guid projectId)
+ {
+ try
+ {
+ return ProjectSidecars.ReadRecord(folder.ProjectRecordPath(projectId));
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ throw new ProjectsUnreadableException($"The record of project {projectId} cannot be read now: {ex.Message}");
+ }
+ }
+
///
/// Writes Main's into the sequence: its positions placed on the sequence's frames, then
- /// project.json and the index's settings (and hero frame) with the settings version as the check. Recorded as
- /// mirrored when the index took it; a settings edit that won the race is caught up by the next operation (merged).
+ /// project.json , Main's record naming it as mirrored, and the index's settings (and hero frame) with the settings
+ /// version as the check. Recorded as mirrored when the index took it; a settings edit that won the race is caught up by
+ /// the next operation (merged). A version this Chronos cannot read (of a newer one) is never mirrored: nothing is written.
///
private async Task MirrorAsync(LeaseContext context, SequenceProject main, ProjectVersion version, CancellationToken cancellationToken)
{
var frames = await LoadFramesAsync(context.Row.Id, cancellationToken).ConfigureAwait(false);
- var placed = ProjectSettings.FromJson(ProjectContent.Place(version.SettingsJson, frames.Photos));
+ ProjectSettings placed;
+ try
+ {
+ if (IsNewer(version)) throw new InvalidDataException($"Version {version.Number} of Main is of a newer Chronos.");
+ placed = ProjectSettings.FromJson(ProjectContent.Place(version.SettingsJson, frames.Photos));
+ }
+ catch (Exception ex) when (ex is JsonException or InvalidDataException)
+ {
+ Log.Warning("projects", $"Version {version.Number} of Main of sequence {context.Row.Id} cannot be mirrored into its settings; they stay as they are", ex);
+ return;
+ }
var json = placed.ToJson();
Directory.CreateDirectory(context.Folder.MetaDirectory);
AtomicFile.WriteAllText(context.Folder.ProjectPath, json);
+ try
+ {
+ var record = ProjectSidecars.ReadRecord(context.Folder.ProjectRecordPath(main.Id));
+ ProjectSidecars.WriteRecord(context.Folder.ProjectRecordPath(main.Id), record with { Mirrored = new ProjectHead(version.Id, version.Number) });
+ }
+ catch (Exception ex) when (ex is IOException or InvalidDataException or UnauthorizedAccessException)
+ {
+ // An index meeting Main then compares project.json with the versions the head descends from.
+ Log.Warning("projects", $"The record of Main of sequence {context.Row.Id} could not name version {version.Number} as mirrored", ex);
+ }
var hero = placed.HeroFrame is { } index && (uint)index < (uint)frames.Frames.Count ? frames.Frames[index].Id : Guid.Empty;
var result = await _db.Sequences.EditAsync(context.Row.Id, new SequenceEdit { SettingsJson = json, HeroFrameId = hero }, context.Row.SettingsVersion,
context.Row.FramesVersion, cancellationToken).ConfigureAwait(false);
@@ -536,6 +895,20 @@ public sealed class ProjectService
}
}
+ /// Whether settings this Chronos reads strictly (a member of the wrong type is refused): a version that can be mirrored and edited.
+ private static bool Readable(string json)
+ {
+ try
+ {
+ ProjectSettings.FromJson(json);
+ return true;
+ }
+ catch (Exception ex) when (ex is JsonException or InvalidDataException)
+ {
+ return false;
+ }
+ }
+
/// Whether settings name no frame by position (every position parked by photo).
private static bool IsPortable(string json)
{
@@ -592,10 +965,11 @@ public sealed class ProjectService
private static ProjectException SequenceMissing(Guid sequenceId) => new(ProjectError.SequenceNotFound, $"Sequence {sequenceId} is not in the index.");
- /// The sequence as read under its lease (the mirror moves its settings version) and its folder.
- private sealed class LeaseContext(SequencePaths folder, Sequence row)
+ /// The sequence as read under its lease (the mirror moves its settings version), its folder, and what its projects' reconcile met.
+ private sealed class LeaseContext(SequencePaths folder, Sequence row, ProjectReconcile reconcile)
{
public SequencePaths Folder { get; } = folder;
public Sequence Row { get; set; } = row;
+ public ProjectReconcile Reconcile { get; } = reconcile;
}
}
diff --git a/src/Chronos.Data/Projects/ProjectSidecars.cs b/src/Chronos.Data/Projects/ProjectSidecars.cs
index 24d50d17..34f5cafc 100644
--- a/src/Chronos.Data/Projects/ProjectSidecars.cs
+++ b/src/Chronos.Data/Projects/ProjectSidecars.cs
@@ -11,20 +11,26 @@ namespace Chronos.Data.Projects;
/// The newest version of a project as its record names it: id and number.
public sealed record ProjectHead(Guid Id, int Number);
+/// A version's file would be over , so it could never be read back: nothing was written.
+public sealed class ProjectVersionTooLargeException(string message) : IOException(message);
+
///
/// The record of a project in its folder (.chronos/projects/<id>/project.json ): what the index row is rebuilt
-/// from. is set when the project was deleted, before its folder moves to the projects' trash.
+/// from. is set when the project was deleted, once its folder is in the projects' trash.
///
+/// Main only: the version whose settings a mirror last wrote into the sequence's project.json
+/// (any index of the archive). An index that meets Main for the first time compares project.json with it, never
+/// with a copy of the settings it may have read long ago ( ). Older records have none.
public sealed record ProjectRecord(Guid Id, Guid SequenceId, string Name, bool IsMain, ProjectHead? Head, DateTimeOffset Created, string? CreatedBy,
- DateTimeOffset Updated, DateTimeOffset? Deleted = null)
+ DateTimeOffset Updated, DateTimeOffset? Deleted = null, ProjectHead? Mirrored = null)
{
- /// The record of an index row.
- public static ProjectRecord Of(SequenceProject project)
+ /// The record of an index row, naming as the version mirrored last (Main).
+ public static ProjectRecord Of(SequenceProject project, ProjectHead? mirrored = null)
{
ArgumentNullException.ThrowIfNull(project);
return new ProjectRecord(project.Id, project.SequenceId, project.Name, project.IsMain,
project.HeadVersionId is { } head ? new ProjectHead(head, project.HeadNumber) : null, project.CreatedAt, project.CreatedBy, project.UpdatedAt,
- project.DeletedAt);
+ project.DeletedAt, project.IsMain ? mirrored : null);
}
/// The index row of this record (nothing mirrored known yet).
@@ -58,20 +64,33 @@ public static class ProjectSidecars
public const string VersionFormat = "chronos-project-version";
public const int FormatVersion = 1;
- /// A version file is a few kilobytes; a larger one is not read.
- public const int MaxVersionBytes = 8 << 20;
+ ///
+ /// The one size limit of a version file, for writing and reading alike: a version that could not be read back is never
+ /// written ( ). A version is a few kilobytes; each photo it parks (an exclusion, the hero, a
+ /// keyframe waiting for its photo) takes about 340 bytes of the file, so this holds the positions of about 190,000
+ /// photos, well past the largest sequences.
+ ///
+ public const int MaxVersionBytes = 64 << 20;
private static readonly JsonWriterOptions Indented = new() { Indented = true };
+ /// Whether can be stored: its file within .
+ public static bool Fits(ProjectVersion version) => Encoding.UTF8.GetByteCount(VersionJson(version)) <= MaxVersionBytes;
+
///
/// Writes a version file. A version is immutable: an existing file is left when it holds the same version (same id
/// and content hash) and refused otherwise, unless (a workspace's own copy of a version,
/// stored at its home since).
///
/// Another version is at that path.
+ /// The file would be over : nothing was written.
public static void WriteVersion(string path, ProjectVersion version, bool replace = false)
{
ArgumentNullException.ThrowIfNull(version);
+ var json = VersionJson(version);
+ var bytes = Encoding.UTF8.GetByteCount(json);
+ if (bytes > MaxVersionBytes)
+ throw new ProjectVersionTooLargeException($"Version {version.Number} of project {version.ProjectId} takes {bytes} bytes, over the {MaxVersionBytes} a version may.");
if (!replace && File.Exists(path))
{
ProjectVersion existing;
@@ -92,11 +111,11 @@ public static class ProjectSidecars
}
}
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
- AtomicFile.WriteAllText(path, VersionJson(version));
+ AtomicFile.WriteAllText(path, json);
}
/// Reads a version file and checks its content hash.
- /// Not a version file, damaged, newer, or its settings do not match their hash.
+ /// Not a version file, damaged, newer, over , or its settings do not match their hash.
/// The file cannot be read.
public static ProjectVersion ReadVersion(string path)
{
@@ -189,6 +208,13 @@ public static class ProjectSidecars
if (record.CreatedBy is { } by) w.WriteString("createdBy", by);
w.WriteString("updated", record.Updated);
if (record.Deleted is { } deleted) w.WriteString("deleted", deleted);
+ if (record.Mirrored is { } mirrored)
+ {
+ w.WriteStartObject("mirrored");
+ w.WriteString("id", mirrored.Id);
+ w.WriteNumber("number", mirrored.Number);
+ w.WriteEndObject();
+ }
w.WriteEndObject();
}
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
@@ -203,22 +229,26 @@ public static class ProjectSidecars
using var document = Parse(AtomicFile.ReadAllText(path), path);
var root = document.RootElement;
Check(root, RecordFormat, path);
- ProjectHead? head = null;
- if (root.TryGetProperty("head", out var h) && h.ValueKind == JsonValueKind.Object && Id(h, "id") is { } headId && Int(h, "number") is { } number and > 0)
- head = new ProjectHead(headId, number);
var name = Text(root, "name");
return new ProjectRecord(
Id(root, "id") ?? throw new InvalidDataException($"{path} has no project id."),
Id(root, "sequenceId") ?? throw new InvalidDataException($"{path} names no sequence."),
string.IsNullOrWhiteSpace(name) ? throw new InvalidDataException($"{path} has no name.") : name,
root.TryGetProperty("isMain", out var main) && main.ValueKind == JsonValueKind.True,
- head,
+ Head(root, "head"),
Time(root, "created") ?? default,
Text(root, "createdBy"),
Time(root, "updated") ?? Time(root, "created") ?? default,
- Time(root, "deleted"));
+ Time(root, "deleted"),
+ Head(root, "mirrored"));
}
+ /// A version a record names (its head, the version mirrored): { "id", "number" } ; null when it names none.
+ private static ProjectHead? Head(JsonElement root, string name) =>
+ root.TryGetProperty(name, out var h) && h.ValueKind == JsonValueKind.Object && Id(h, "id") is { } id && Int(h, "number") is { } number and > 0
+ ? new ProjectHead(id, number)
+ : null;
+
/// The version files of a folder by number and id (000001-<id>.json ), in number order; other files are not listed.
public static IReadOnlyList<(int Number, Guid Id, string Path)> ListVersions(string directory)
{
diff --git a/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs b/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs
new file mode 100644
index 00000000..4bc6ce30
--- /dev/null
+++ b/tests/Chronos.Data.Tests/Projects/ProjectRecoveryTests.cs
@@ -0,0 +1,502 @@
+using System.Security.Cryptography;
+using Chronos.Core.Archiving;
+using Chronos.Core.Model;
+using Chronos.Core.Project;
+using Chronos.Data.Archiving;
+using Chronos.Data.Projects;
+using Chronos.Data.Tests.Archiving;
+using Chronos.Data.Tests.Infrastructure;
+using Chronos.Testing;
+using static Chronos.Data.Tests.Archiving.ArchiveFixture;
+using static Chronos.Data.Tests.Projects.ProjectServiceTests;
+
+namespace Chronos.Data.Tests.Projects;
+
+///
+/// The sidecars stay the truth of a sequence's projects when they cannot be read, are missing, are a newer Chronos's, or
+/// sit on a read-only archive, and an index that meets them late never takes its own old copy for the sequence's settings.
+///
+[Category("projects")]
+public abstract class ProjectRecoveryTests(TestContext context) : DatabaseTest(context)
+{
+ private async Task FixtureAsync() => Own(await CreateAsync(Context, database: await NewOptionsAsync(multiConnection: true)));
+
+ /// The live project folders of a sequence (one per project).
+ private static string[] ProjectFolders(SequencePaths folder) =>
+ [.. Directory.GetDirectories(folder.ProjectsDirectory).Where(d => Guid.TryParseExact(Path.GetFileName(d), "N", out _))];
+
+ private static int MainRecords(SequencePaths folder) =>
+ ProjectFolders(folder).Count(d => ProjectSidecars.ReadRecord(Path.Combine(d, "project.json")).IsMain);
+
+ private static ProjectSettings Settings(ArchiveFixture fx, Sequence sequence) =>
+ ProjectSettings.FromJson(File.ReadAllText(fx.Paths.Sequence(sequence.Folder).ProjectPath));
+
+ [Test]
+ public async Task ARecordThatCannotBeReadNeverTakesItsProjectForGoneNorMakesASecondMain()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
+ var folder = fx.Paths.Sequence(sequence.Folder);
+ var mainRecord = File.ReadAllText(folder.ProjectRecordPath(main.Id));
+ var verticalRecord = File.ReadAllText(folder.ProjectRecordPath(vertical.Id));
+
+ // Main's record as a newer Chronos writes it, Vertical's damaged.
+ var newer = mainRecord.Replace("\"formatVersion\": 1", "\"formatVersion\": 2", StringComparison.Ordinal);
+ File.WriteAllText(folder.ProjectRecordPath(main.Id), newer);
+ File.WriteAllText(folder.ProjectRecordPath(vertical.Id), "{ not json");
+ Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
+ Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id);
+ Assert.Count(2, ProjectFolders(folder));
+ Assert.Null((await fx.Db.Projects.GetAsync(vertical.Id))!.DeletedAt);
+ // Nothing is written over a record that cannot be read: the versions sent wait.
+ var head = await projects.HeadAsync(vertical.Id, fx.Ct);
+ await Assert.ThrowsAsync(() => projects.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct));
+ var home = new ArchiveProjectHome(fx.Db, projects, sequence.Id);
+ await Assert.ThrowsAsync(() => home.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct));
+ await Assert.ThrowsAsync(() => projects.RenameAsync(vertical.Id, "Square", fx.Ct));
+ await Assert.ThrowsAsync(() => projects.DeleteAsync(vertical.Id, fx.Ct));
+ Assert.Equal("{ not json", File.ReadAllText(folder.ProjectRecordPath(vertical.Id)));
+ Assert.Equal(newer, File.ReadAllText(folder.ProjectRecordPath(main.Id)));
+
+ // Readable again: everything is as it was, and goes on.
+ File.WriteAllText(folder.ProjectRecordPath(main.Id), mainRecord);
+ File.WriteAllText(folder.ProjectRecordPath(vertical.Id), verticalRecord);
+ Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
+ Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
+
+ // An index that meets Main's record unreadable makes no Main of its own either, and takes it once it reads.
+ File.WriteAllText(folder.ProjectRecordPath(main.Id), newer);
+ await fx.ReopenWithEmptyIndexAsync();
+ Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
+ var rebuilt = fx.Service.Projects;
+ Assert.SequenceEqual([vertical.Id], (await rebuilt.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
+ await Assert.ThrowsAsync(() => rebuilt.MainAsync(sequence.Id, fx.Ct));
+ await Assert.ThrowsAsync(() => new ArchiveProjectHome(fx.Db, rebuilt, sequence.Id).ListAsync(fx.Ct));
+ Assert.Count(2, ProjectFolders(folder));
+ File.WriteAllText(folder.ProjectRecordPath(main.Id), mainRecord);
+ Assert.Equal(main.Id, (await rebuilt.MainAsync(sequence.Id, fx.Ct)).Id);
+ Assert.Equal(1, MainRecords(folder));
+ }
+
+ [Test]
+ public async Task AMissingProjectsFolderIsWrittenAgainFromTheIndex()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
+ var pushed = (await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [2] }, frames),
+ fx.Ct)).Version!;
+ var folder = fx.Paths.Sequence(sequence.Folder);
+
+ // Moved away for a moment (a sync tool, a backup put back without it): no project is taken for gone, none made anew.
+ Directory.Move(folder.ProjectsDirectory, folder.ProjectsDirectory + "-aside");
+ Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
+ Assert.Equal(1, MainRecords(folder));
+ Assert.Equal(pushed.Id, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Head!.Id);
+ Assert.Equal(pushed.ContentHash, ProjectSidecars.ReadVersion(folder.VersionPath(vertical.Id, 2, pushed.Id)).ContentHash);
+ Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, Draft(pushed, new ProjectSettings(), frames), fx.Ct)).Outcome);
+
+ // The sidecars written again are the truth again: a rebuilt index has every project and version back.
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.ScanAsync(ScanMode.Rebuild);
+ var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct);
+ Assert.SequenceEqual([main.Id, vertical.Id], listed.Select(p => p.Id));
+ Assert.Equal(3, listed[1].HeadNumber);
+ }
+
+ [Test]
+ public async Task AMainAnOlderChronosTookForGoneIsListedAgain()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, _) = await UploadedAsync(fx, 2);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var folder = fx.Paths.Sequence(sequence.Folder);
+
+ // Its live record says it is not deleted: Main again, no second one.
+ await fx.Db.Projects.UpdateAsync((await fx.Db.Projects.GetAsync(main.Id))! with { DeletedAt = DateTimeOffset.UtcNow });
+ Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
+ Assert.Equal(main.Id, (await projects.MainAsync(sequence.Id, fx.Ct)).Id);
+
+ // And with its folder gone too: written again from the index.
+ await fx.Db.Projects.UpdateAsync((await fx.Db.Projects.GetAsync(main.Id))! with { DeletedAt = DateTimeOffset.UtcNow });
+ Directory.Move(folder.ProjectDirectory(main.Id), folder.ProjectDirectory(main.Id) + "-aside");
+ Assert.SequenceEqual([main.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
+ Assert.True(ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).IsMain);
+ Assert.Equal(1, MainRecords(folder));
+ }
+
+ [Test]
+ public async Task ASecondIndexMeetingMainBetweenTwoPushesRevertsNothing()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var second = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct),
+ new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 0.7 } }, frames), fx.Ct)).Version!;
+
+ // Another PC indexes the archive (its copy of the settings: version 2's), then this one pushes version 3.
+ var (other, _) = await fx.OtherWriterAsync();
+ var third = (await projects.PushAsync(main.Id, Draft(second, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 1.4 } }, frames),
+ fx.Ct)).Version!;
+
+ // Its first look takes the settings from the archive, not from its old copy: no version is added.
+ await other.Projects.ListAsync(sequence.Id, fx.Ct);
+ Assert.SequenceEqual([3, 2, 1], (await other.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number));
+ Assert.Equal(1.4, Settings(fx, sequence).Exposure.ExposureOffsetEv);
+ Assert.Equal(third.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id);
+ Assert.Equal(1.4, ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).Exposure.ExposureOffsetEv);
+ }
+
+ [Test]
+ public async Task AnIndexWithAnOldCopyOfTheSettingsBringsItUpToMain()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var pushed = (await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [0] }, frames),
+ fx.Ct)).Version!;
+
+ // An index taken back from a backup made before: it holds the sequence with the settings it had then, and no projects.
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.ScanAsync();
+ await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [1] });
+ var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct);
+ Assert.Equal((pushed.Id, 2), (listed[0].HeadVersionId, listed[0].HeadNumber));
+ Assert.SequenceEqual([2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number));
+ Assert.SequenceEqual([0], ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).ExcludedFrames);
+ Assert.SequenceEqual([0], Settings(fx, sequence).ExcludedFrames);
+ }
+
+ [Test]
+ public async Task AMirrorACrashCutShortBeforeProjectJsonIsCompletedAfterARebuild()
+ {
+ foreach (var olderRecord in new[] { false, true })
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var first = await projects.HeadAsync(main.Id, fx.Ct);
+ await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { Stack = new StackSettings { Mode = StackSetting.Median } }, frames), fx.Ct);
+
+ // As if the push had stopped before its mirror wrote project.json (an older record names no mirror at all).
+ var folder = fx.Paths.Sequence(sequence.Folder);
+ File.WriteAllText(folder.ProjectPath, ProjectSettings.FromJson(ProjectContent.Place(first.SettingsJson, Photos(frames))).ToJson());
+ var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id));
+ ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Mirrored = olderRecord ? null : new ProjectHead(first.Id, 1) });
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.ScanAsync(ScanMode.Rebuild);
+
+ Assert.SequenceEqual([2, 1], (await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct)).Select(v => v.Number));
+ Assert.Equal(StackSetting.Median, Settings(fx, sequence).Stack.Mode);
+ Assert.Equal(StackSetting.Median, ProjectSettings.FromJson((await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson!).Stack.Mode);
+ }
+ }
+
+ [Test]
+ public async Task SettingsAnOlderChronosWroteIntoProjectJsonBecomeAnExternalVersionForAnIndexMeetingMain()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ await projects.PushAsync(main.Id, Draft(await projects.HeadAsync(main.Id, fx.Ct), new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = 0.7 } },
+ frames), fx.Ct);
+
+ // An older Chronos on another PC edits the sequence (project.json); this PC's index is then rebuilt.
+ File.WriteAllText(fx.Paths.Sequence(sequence.Folder).ProjectPath, new ProjectSettings { Exposure = new ExposureSettings { ExposureOffsetEv = -1 } }.ToJson());
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.ScanAsync(ScanMode.Rebuild);
+ var history = await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct);
+ Assert.SequenceEqual([3, 2, 1], history.Select(v => v.Number));
+ Assert.Equal(ProjectVersionKind.External, history[0].Kind);
+ Assert.Equal(-1, ProjectSettings.FromJson(history[0].SettingsJson).Exposure.ExposureOffsetEv);
+ }
+
+ [Test]
+ public async Task AMainHeadOfANewerChronosIsListedAndPulledButNeverMirroredNorBuiltOn()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
+ var head = await projects.HeadAsync(main.Id, fx.Ct);
+ var folder = fx.Paths.Sequence(sequence.Folder);
+
+ // A newer Chronos on another PC sharing the archive pushes version 2 in its settings format, and mirrors it.
+ var settings = """{ "version": 2, "exposure": { "exposureOffsetEv": 0.5 }, "futureThing": true }""";
+ var now = DateTimeOffset.UtcNow;
+ var future = new ProjectVersion
+ {
+ Id = Guid.CreateVersion7(),
+ ProjectId = main.Id,
+ ParentId = head.Id,
+ Number = 2,
+ Kind = ProjectVersionKind.Commit,
+ ClientName = "newer",
+ SettingsJson = settings,
+ ContentHash = ProjectContent.Hash(settings),
+ SettingsFormat = 2,
+ FramesVersion = head.FramesVersion,
+ CreatedAt = now,
+ StoredAt = now,
+ };
+ ProjectSidecars.WriteVersion(folder.VersionPath(main.Id, 2, future.Id), future);
+ var record = ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id));
+ ProjectSidecars.WriteRecord(folder.ProjectRecordPath(main.Id), record with { Head = new ProjectHead(future.Id, 2), Mirrored = new ProjectHead(future.Id, 2) });
+ File.WriteAllText(folder.ProjectPath, settings);
+ var copy = (await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson;
+
+ Assert.SequenceEqual([main.Id, vertical.Id], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Id));
+ Assert.Equal(future.Id, (await projects.HeadAsync(main.Id, fx.Ct)).Id);
+ Assert.Equal(future.Id, (await new ArchiveProjectHome(fx.Db, projects, sequence.Id).VersionsAfterAsync(main.Id, 1, 10, fx.Ct)).Single().Id);
+ Assert.Equal(PushOutcome.Stored,
+ (await projects.PushAsync(vertical.Id, Draft(await projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
+ Assert.Count(2, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
+ Assert.Equal(settings, File.ReadAllText(folder.ProjectPath));
+ Assert.Equal(copy, (await fx.Db.Sequences.GetAsync(sequence.Id))!.SettingsJson);
+
+ // An older path writes the settings here meanwhile: nothing is merged into, or added over, the newer head.
+ await WriteSettingsAsync(fx, sequence.Id, new ProjectSettings { ExcludedFrames = [2] });
+ await projects.ListAsync(sequence.Id, fx.Ct);
+ await projects.ListAsync(sequence.Id, fx.Ct);
+ Assert.Count(2, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
+ Assert.Equal(settings, File.ReadAllText(folder.ProjectPath));
+
+ // Nor by an index meeting it for the first time.
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.ScanAsync(ScanMode.Rebuild);
+ Assert.Equal(future.Id, (await fx.Service.Projects.HeadAsync(main.Id, fx.Ct)).Id);
+ Assert.Count(2, await fx.Service.Projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
+ Assert.Equal(settings, File.ReadAllText(folder.ProjectPath));
+ }
+
+ [Test]
+ public async Task APushThisChronosCannotReadStrictlyIsRefused()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 2);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var head = await projects.HeadAsync(main.Id, fx.Ct);
+ var wrong = """{ "exposure": 5 }""";
+ var result = await projects.PushAsync(main.Id, Draft(head, new ProjectSettings(), frames) with { SettingsJson = wrong, ContentHash = ProjectContent.Hash(wrong) },
+ fx.Ct);
+ Assert.Equal(PushOutcome.Invalid, result.Outcome);
+ Assert.Count(1, await projects.HistoryAsync(main.Id, cancellationToken: fx.Ct));
+ Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(main.Id, Draft(head, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Outcome);
+ }
+
+ [Test]
+ public async Task AHeadThisChronosCannotReadLeavesTheRecordAndTheHistoryUntouched()
+ {
+ foreach (var missing in new[] { false, true })
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var first = await projects.HeadAsync(main.Id, fx.Ct);
+ var second = (await projects.PushAsync(main.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!;
+ var folder = fx.Paths.Sequence(sequence.Folder);
+
+ // Version 2's file in a newer Chronos's format, or not here yet (a sync tool that brought the record first).
+ var path = folder.VersionPath(main.Id, 2, second.Id);
+ var text = File.ReadAllText(path);
+ if (missing) File.Delete(path);
+ else File.WriteAllText(path, text.Replace("\"formatVersion\": 1", "\"formatVersion\": 2", StringComparison.Ordinal));
+ var recordBytes = File.ReadAllBytes(folder.ProjectRecordPath(main.Id));
+ await fx.ReopenWithEmptyIndexAsync();
+ Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
+ var rebuilt = fx.Service.Projects;
+
+ var listed = await rebuilt.ListAsync(sequence.Id, fx.Ct);
+ Assert.Equal((main.Id, first.Id), (listed.Single().Id, listed.Single().HeadVersionId));
+ Assert.Equal(first.Id, (await rebuilt.HeadAsync(main.Id, fx.Ct)).Id);
+ await Assert.ThrowsAsync(() => rebuilt.PushAsync(main.Id, Draft(first, new ProjectSettings(), frames), fx.Ct));
+ Assert.SequenceEqual(recordBytes, File.ReadAllBytes(folder.ProjectRecordPath(main.Id)));
+ Assert.False(Directory.Exists(Path.Combine(folder.ProjectVersionsDirectory(main.Id), ".stray")));
+ Assert.Count(missing ? 1 : 2, ProjectSidecars.ListVersions(folder.ProjectVersionsDirectory(main.Id)));
+ Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
+
+ // Once it reads, the history goes on from it.
+ File.WriteAllText(path, text);
+ Assert.Equal(second.Id, (await rebuilt.HeadAsync(main.Id, fx.Ct)).Id);
+ Assert.Equal(PushOutcome.Stored, (await rebuilt.PushAsync(main.Id, Draft(second, new ProjectSettings(), frames), fx.Ct)).Outcome);
+ }
+ }
+
+ [Test]
+ public async Task VersionFilesOfOneNumberAndRecordsThatAllSayMainNeverStopARebuild()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 3);
+ var other = (await UploadedAsync(fx, 1, seed: 300)).Sequence;
+ var projects = fx.Service.Projects;
+ var main = await projects.MainAsync(sequence.Id, fx.Ct);
+ var otherMain = await projects.MainAsync(other.Id, fx.Ct);
+ var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
+ var first = await projects.HeadAsync(vertical.Id, fx.Ct);
+ var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [2] }, frames), fx.Ct)).Version!;
+ var folder = fx.Paths.Sequence(sequence.Folder);
+
+ // Another file numbered 2, off the record's chain, and two more records that say they are Main and are named so.
+ var twin = Draft(first, new ProjectSettings { ExcludedFrames = [0] }, frames) with { StoredAt = DateTimeOffset.UtcNow };
+ ProjectSidecars.WriteVersion(folder.VersionPath(vertical.Id, 2, twin.Id), twin);
+ foreach (var _ in new[] { 1, 2 })
+ {
+ var id = Guid.CreateVersion7();
+ ProjectSidecars.WriteRecord(folder.ProjectRecordPath(id), new ProjectRecord(id, sequence.Id, ProjectService.MainName, true, null, DateTimeOffset.UtcNow, null,
+ DateTimeOffset.UtcNow));
+ }
+ await fx.ReopenWithEmptyIndexAsync();
+ Assert.Empty((await fx.ScanAsync(ScanMode.Rebuild)).Failed);
+
+ var listed = await fx.Service.Projects.ListAsync(sequence.Id, fx.Ct);
+ Assert.Equal(main.Id, listed.Single(p => p.IsMain).Id);
+ Assert.SequenceEqual(["Main", "Main (2)", "Main (3)", "Vertical"], listed.Select(p => p.Name).Order(StringComparer.Ordinal));
+ Assert.Equal(second.Id, listed.Single(p => p.Id == vertical.Id).HeadVersionId);
+ Assert.Count(1, Directory.GetFiles(Path.Combine(folder.ProjectVersionsDirectory(vertical.Id), ".stray")));
+ Assert.NotNull(await fx.Db.Projects.GetAsync(otherMain.Id));
+ }
+
+ [Test]
+ public async Task ALargeVersionIsStoredAndReadBackAndOneTooLargeIsNeverStored()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 2);
+ var projects = fx.Service.Projects;
+ var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
+ var head = await projects.HeadAsync(vertical.Id, fx.Ct);
+ var folder = fx.Paths.Sequence(sequence.Folder);
+
+ // Some 30,000 photos parked by a long sequence (its nights excluded): past the old 8 MB read limit.
+ var pending = Enumerable.Range(0, 30_000).Select(i => new PendingFrame
+ {
+ Sha256 = Convert.ToHexStringLower(SHA256.HashData(BitConverter.GetBytes(i))),
+ Size = 1_000_000 + i,
+ CapturedAt = Start.AddSeconds(i),
+ Excluded = true,
+ }).ToList();
+ var settings = ProjectContent.Canonical(new ProjectSettings { PendingFrames = pending }.ToJson());
+ var large = Draft(head, new ProjectSettings(), frames) with { SettingsJson = settings, ContentHash = ProjectContent.Hash(settings) };
+ Assert.Equal(PushOutcome.Stored, (await projects.PushAsync(vertical.Id, large, fx.Ct)).Outcome);
+ var path = folder.VersionPath(vertical.Id, 2, large.Id);
+ Assert.Greater(new FileInfo(path).Length, 8L << 20);
+ Assert.Equal(large.ContentHash, ProjectSidecars.ReadVersion(path).ContentHash);
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.ScanAsync(ScanMode.Rebuild);
+ var rebuilt = fx.Service.Projects;
+ Assert.Equal(large.Id, (await rebuilt.HeadAsync(vertical.Id, fx.Ct)).Id);
+
+ // One whose file would be past the limit is refused, and nothing is written.
+ var huge = Draft(large, new ProjectSettings(), frames) with { Note = new string('x', ProjectSidecars.MaxVersionBytes) };
+ Assert.Equal(PushOutcome.TooLarge, (await rebuilt.PushAsync(vertical.Id, huge, fx.Ct)).Outcome);
+ Assert.False(File.Exists(folder.VersionPath(vertical.Id, 3, huge.Id)));
+ Assert.Throws(() => ProjectSidecars.WriteVersion(Path.Combine(Context.CreateTempDirectory("huge"), "v.json"), huge));
+ }
+
+ [Test]
+ public async Task TheHomeHoldsAVersionOnlyWhileItsFileReadsBack()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, frames) = await UploadedAsync(fx, 2);
+ var projects = fx.Service.Projects;
+ var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
+ var first = await projects.HeadAsync(vertical.Id, fx.Ct);
+ var second = (await projects.PushAsync(vertical.Id, Draft(first, new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!;
+ var folder = fx.Paths.Sequence(sequence.Folder);
+ IProjectHome home = new ArchiveProjectHome(fx.Db, projects, sequence.Id);
+
+ Assert.True(await home.HoldsAsync(vertical.Id, first, fx.Ct));
+ Assert.True(await home.HoldsAsync(vertical.Id, second, fx.Ct));
+ Assert.False(await home.HoldsAsync(vertical.Id, second with { ContentHash = new string('0', 64) }, fx.Ct));
+ // The index still has both rows, but the archive no longer holds them as they were.
+ File.WriteAllText(folder.VersionPath(vertical.Id, 1, first.Id), "damaged");
+ File.Delete(folder.VersionPath(vertical.Id, 2, second.Id));
+ Assert.NotNull(await home.GetVersionAsync(vertical.Id, first.Id, fx.Ct));
+ Assert.False(await home.HoldsAsync(vertical.Id, first, fx.Ct));
+ Assert.False(await home.HoldsAsync(vertical.Id, second, fx.Ct));
+ }
+
+ [Test]
+ public async Task ADeleteThatCannotMoveTheFolderChangesNothing()
+ {
+ var fx = await FixtureAsync();
+ var (sequence, _) = await UploadedAsync(fx, 2);
+ var projects = fx.Service.Projects;
+ var vertical = await projects.CreateAsync(sequence.Id, "Vertical", cancellationToken: fx.Ct);
+ var folder = fx.Paths.Sequence(sequence.Folder);
+
+ // The projects' trash cannot take the folder (a file in its way): the delete fails and says so, and nothing changed.
+ File.WriteAllText(folder.ProjectsTrashDirectory, "in the way");
+ await Assert.ThrowsAsync(() => projects.DeleteAsync(vertical.Id, fx.Ct));
+ Assert.Null(ProjectSidecars.ReadRecord(folder.ProjectRecordPath(vertical.Id)).Deleted);
+ Assert.SequenceEqual(["Main", "Vertical"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name));
+
+ File.Delete(folder.ProjectsTrashDirectory);
+ await projects.DeleteAsync(vertical.Id, fx.Ct);
+ Assert.False(Directory.Exists(folder.ProjectDirectory(vertical.Id)));
+ Assert.NotNull(ProjectSidecars.ReadRecord(Path.Combine(Directory.GetDirectories(folder.ProjectsTrashDirectory).Single(), "project.json")).Deleted);
+ Assert.SequenceEqual(["Main"], (await projects.ListAsync(sequence.Id, fx.Ct)).Select(p => p.Name));
+ }
+
+ [Test]
+ public async Task ASequenceOfAReadOnlyArchiveOpensFromItsSidecarsWithoutWriting()
+ {
+ var fx = await FixtureAsync();
+ var (fresh, freshFrames) = await UploadedAsync(fx, 2);
+ var (edited, frames) = await UploadedAsync(fx, 2, seed: 300);
+ var vertical = await fx.Service.Projects.CreateAsync(edited.Id, "Vertical", cancellationToken: fx.Ct);
+ var pushed = (await fx.Service.Projects.PushAsync(vertical.Id,
+ Draft(await fx.Service.Projects.HeadAsync(vertical.Id, fx.Ct), new ProjectSettings { ExcludedFrames = [1] }, frames), fx.Ct)).Version!;
+ await fx.ReopenWithEmptyIndexAsync();
+ await fx.ScanAsync();
+ var projects = fx.Service.Projects;
+ projects.Availability = _ => ArchiveAvailability.ReadOnly;
+ var locks = Directory.Exists(fx.Paths.LocksDirectory) ? Directory.GetFiles(fx.Paths.LocksDirectory).Length : 0;
+
+ // A sequence with projects: read from its sidecars as they are.
+ var home = new ArchiveProjectHome(fx.Db, projects, edited.Id);
+ Assert.SequenceEqual(["Main", "Vertical"], (await home.ListAsync(fx.Ct)).Select(p => p.Name));
+ Assert.Equal(pushed.Id, (await home.HeadAsync(vertical.Id, fx.Ct)).Id);
+ await Assert.ThrowsAsync(() => home.PushAsync(vertical.Id, Draft(pushed, new ProjectSettings(), frames), fx.Ct));
+
+ // A sequence with none yet: a Main in the index only, nothing written to the archive.
+ var freshHome = new ArchiveProjectHome(fx.Db, projects, fresh.Id);
+ var main = (await freshHome.ListAsync(fx.Ct)).Single();
+ Assert.True(main.IsMain);
+ var head = await freshHome.HeadAsync(main.Id, fx.Ct);
+ Assert.False(Directory.Exists(fx.Paths.Sequence(fresh.Folder).ProjectsDirectory));
+ Assert.Equal(locks, Directory.Exists(fx.Paths.LocksDirectory) ? Directory.GetFiles(fx.Paths.LocksDirectory).Length : 0);
+ await Assert.ThrowsAsync(() => freshHome.PushAsync(main.Id, Draft(head, new ProjectSettings(), freshFrames), fx.Ct));
+
+ // Once the archive takes writes, that Main's sidecars are written, under the same id, and it goes on.
+ projects.Availability = ArchiveVolume.Probe;
+ Assert.Equal(main.Id, (await freshHome.ListAsync(fx.Ct)).Single().Id);
+ var folder = fx.Paths.Sequence(fresh.Folder);
+ Assert.Equal((main.Id, true), (ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).Id, ProjectSidecars.ReadRecord(folder.ProjectRecordPath(main.Id)).IsMain));
+ Assert.FileExists(folder.VersionPath(main.Id, 1, head.Id));
+ Assert.Equal(PushOutcome.Stored, (await freshHome.PushAsync(main.Id, Draft(head, new ProjectSettings { ExcludedFrames = [0] }, freshFrames), fx.Ct)).Outcome);
+ }
+}
+
+public sealed class SqliteProjectRecoveryTests(TestContext context) : ProjectRecoveryTests(context)
+{
+ protected override Backend Backend => Backend.SqliteFile;
+}
+
+[RequiresEnvironment(PgTestServer.Variable)]
+public sealed class PostgresProjectRecoveryTests(TestContext context) : ProjectRecoveryTests(context)
+{
+ protected override Backend Backend => Backend.Postgres;
+}