CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
The owner renamed the product and moved the repository to the self-hosted Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control, and asked for Actions and a first automatic release with the server package and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md). - Chronos everywhere: a byte-exact rename of contents and paths, then a per-language review of the forms the old name's ending had dictated. - Gitea instead of GitHub: the update check, licence, issue and support links read ChronosReleasesUrl; the update check takes the highest stable version, offers a pre-release build its final release and reports a repository it cannot see as a failed check. EULA, security policy and Gitea issue forms live at the repository root. - Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release, nightly image and performance gates in the .NET SDK container with git checkout and the system SQLite. The release cross-compiles the Windows programs, checks their icon, version and DPI manifest, packs installers and zips, publishes the Gitea release and pushes the server image (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows runner (WINDOWS_RUNNER). - New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the docker-exec script, which Gitea's service containers do not allow; the throw-away server runs without durability) and pe-check. - Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR, 3 on the shared runner; the weekly performance gates keep theirs). BREAKING CHANGE: programs, settings, data folders, file types, environment variables and the Docker image use the new name; nothing is migrated from Titano, which never had a release. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
280 lines
16 KiB
C#
280 lines
16 KiB
C#
using Chronos.Core.Archiving;
|
|
using Chronos.Core.Diagnostics;
|
|
using Chronos.Core.Model;
|
|
using Chronos.Data.Repositories;
|
|
|
|
namespace Chronos.Data.Archiving;
|
|
|
|
/// <summary>Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original.</summary>
|
|
public sealed record DuplicateGroup(string Sha256, IReadOnlyList<DuplicateFrame> Frames);
|
|
|
|
/// <summary>Index and disk out of step (BRIEF §12.5 Orphans).</summary>
|
|
/// <param name="Sequences">Per indexed sequence: photos the sidecar does not list and frames whose file is gone.</param>
|
|
/// <param name="MissingSequences">Index rows whose folder holds no sidecar any more.</param>
|
|
/// <param name="UnindexedFolders">Folders with photos or a sidecar that the index does not know yet (a scan adopts them).</param>
|
|
public sealed record OrphanReport(IReadOnlyList<SequenceOrphans> Sequences, IReadOnlyList<SequenceLocation> MissingSequences, IReadOnlyList<string> UnindexedFolders)
|
|
{
|
|
public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count);
|
|
public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count);
|
|
}
|
|
|
|
/// <summary>What a maintenance run removed or changed (or would, in a dry run).</summary>
|
|
/// <param name="IndexReclaimable">Bytes of free pages in the SQLite index before the run; null on PostgreSQL (autovacuum) or
|
|
/// when a resumed run had passed that step.</param>
|
|
/// <param name="IndexReclaimed">Bytes the index vacuum gave back; 0 in a dry run or when too little was free to rewrite the file.</param>
|
|
public sealed record MaintenanceReport(
|
|
bool DryRun,
|
|
IReadOnlyList<string> StaleLeases,
|
|
IReadOnlyList<string> TemporaryFiles,
|
|
IReadOnlyList<string> CompactedPacks,
|
|
int DuplicateGroups,
|
|
int DuplicateFlagChanges,
|
|
int UnindexedFiles,
|
|
int MissingFiles,
|
|
long? IndexReclaimable = null,
|
|
long IndexReclaimed = 0);
|
|
|
|
/// <summary>Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched.</summary>
|
|
public sealed class ArchiveMaintenance
|
|
{
|
|
/// <summary>Staged copies older than this belong to no running import.</summary>
|
|
private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
|
|
|
|
/// <summary>Temporary files of atomic writes older than this were left by a crash.</summary>
|
|
private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
|
|
|
|
/// <summary>Packs with more reclaimable bytes than this share are compacted.</summary>
|
|
private const double CompactionRatio = 0.3;
|
|
|
|
/// <summary>The SQLite index is vacuumed when at least this share of it is free pages: rewriting a large index for a
|
|
/// few pages would hold its writers for nothing.</summary>
|
|
internal const double IndexVacuumRatio = 0.1;
|
|
|
|
private readonly ChronosDatabase _db;
|
|
private readonly SequenceWriter _writer;
|
|
private readonly ArchiveIndexer _indexer;
|
|
|
|
internal ArchiveMaintenance(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
|
|
{
|
|
_db = db;
|
|
_writer = writer;
|
|
_indexer = indexer;
|
|
}
|
|
|
|
public async Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
|
|
(await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false))
|
|
.GroupBy(f => f.Sha256, StringComparer.Ordinal)
|
|
.Select(g => new DuplicateGroup(g.Key, g.ToList()))
|
|
.ToList();
|
|
|
|
public async Task<OrphanReport> FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
|
|
var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
// A sequence waiting for its first frame has no folder yet: none of it is missing, and the archive root is not its folder.
|
|
var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false);
|
|
var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList();
|
|
var sequences = new List<SequenceOrphans>();
|
|
var missing = new List<SequenceLocation>();
|
|
foreach (var location in locations)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var sequence = paths.Sequence(location.Folder);
|
|
if (!File.Exists(sequence.ManifestPath))
|
|
{
|
|
missing.Add(location);
|
|
continue;
|
|
}
|
|
var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath));
|
|
if (!orphans.IsEmpty) sequences.Add(orphans);
|
|
}
|
|
var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal);
|
|
var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList();
|
|
return new OrphanReport(sequences, missing, unindexed);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept, and
|
|
/// so are sequences waiting for their first frame (<see cref="ArchiveIngest.AwaitsFolder"/>), which have none yet.
|
|
/// </summary>
|
|
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
|
|
var removed = 0;
|
|
foreach (var id in sequenceIds)
|
|
{
|
|
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
|
|
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath) || ArchiveIngest.AwaitsFolder(sequence, paths)) continue;
|
|
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
|
|
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
return removed;
|
|
}
|
|
|
|
/// <summary>Number of maintenance steps (the job checkpoints after each).</summary>
|
|
internal const int StepCount = 5;
|
|
|
|
/// <summary>Runs the maintenance steps from <paramref name="firstStep"/>: stale leases, leftover temporary files,
|
|
/// thumbnail pack compaction, duplicate flags, index vacuum (SQLite). A dry run only reports.</summary>
|
|
public async Task<MaintenanceReport> RunAsync(Archive archive, bool dryRun, int firstStep, Func<int, ValueTask> stepDone,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var paths = new ArchivePaths(archive.RootPath);
|
|
if (!dryRun) ArchiveVolume.EnsureWritable(paths);
|
|
var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
// A sequence waiting for its first frame has no folder yet: the archive root's pack is compacted under the leases of
|
|
// the root's own sequence, which its thumbnail passes hold, never under another id's.
|
|
var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false);
|
|
var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList();
|
|
IReadOnlyList<string> leases = [], temps = [], packs = [];
|
|
int groups = 0, flagChanges = 0;
|
|
long? reclaimable = null;
|
|
long reclaimed = 0;
|
|
for (var step = firstStep; step < StepCount; step++)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
switch (step)
|
|
{
|
|
case 0:
|
|
leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
|
|
break;
|
|
case 1:
|
|
temps = PurgeTemporaryFiles(paths, locations, dryRun);
|
|
break;
|
|
case 2:
|
|
packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
|
|
break;
|
|
case 3:
|
|
(groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false);
|
|
break;
|
|
case 4:
|
|
(reclaimable, reclaimed) = await VacuumIndexAsync(dryRun, cancellationToken).ConfigureAwait(false);
|
|
break;
|
|
}
|
|
await stepDone(step + 1).ConfigureAwait(false);
|
|
}
|
|
var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false);
|
|
return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles, reclaimable, reclaimed);
|
|
}
|
|
|
|
/// <summary>
|
|
/// The SQLite index's free pages, given back by a vacuum when they are at least <see cref="IndexVacuumRatio"/> of the
|
|
/// file (BRIEF §12.5 "SQLite vacuum"); nothing on PostgreSQL, whose autovacuum does it.
|
|
/// </summary>
|
|
private async Task<(long? Reclaimable, long Reclaimed)> VacuumIndexAsync(bool dryRun, CancellationToken cancellationToken)
|
|
{
|
|
if (await _db.SpaceAsync(cancellationToken).ConfigureAwait(false) is not { } space) return (null, 0);
|
|
if (dryRun || space.ReclaimableBytes == 0 || space.ReclaimableBytes < space.FileBytes * IndexVacuumRatio) return (space.ReclaimableBytes, 0);
|
|
var (before, after) = await _db.VacuumAsync(cancellationToken).ConfigureAwait(false);
|
|
Log.Info("archive", $"Index vacuumed: {before.FileBytes} → {after.FileBytes} bytes");
|
|
return (space.ReclaimableBytes, long.Max(0, before.FileBytes - after.FileBytes));
|
|
}
|
|
|
|
/// <summary>Staging copies and atomic-write temporaries left behind by crashes (never originals).</summary>
|
|
private static List<string> PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun)
|
|
{
|
|
var now = DateTime.UtcNow;
|
|
var found = new List<string>();
|
|
void Collect(string directory, string pattern, TimeSpan age, bool recursive)
|
|
{
|
|
if (!Directory.Exists(directory)) return;
|
|
foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
|
|
{
|
|
if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
|
|
}
|
|
}
|
|
Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
|
|
Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
|
|
foreach (var location in locations)
|
|
{
|
|
var sequence = paths.Sequence(location.Folder);
|
|
Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false);
|
|
Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false);
|
|
}
|
|
if (!dryRun)
|
|
{
|
|
foreach (var file in found)
|
|
{
|
|
try
|
|
{
|
|
File.Delete(file);
|
|
}
|
|
catch (IOException ex)
|
|
{
|
|
Log.Warning("archive", $"Could not remove {file}", ex);
|
|
}
|
|
}
|
|
}
|
|
return found.ConvertAll(paths.ToRelative);
|
|
}
|
|
|
|
private async Task<List<string>> CompactPacksAsync(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun, CancellationToken cancellationToken)
|
|
{
|
|
var compacted = new List<string>();
|
|
foreach (var location in locations)
|
|
{
|
|
var sequence = paths.Sequence(location.Folder);
|
|
long garbage, length;
|
|
using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath))
|
|
{
|
|
garbage = reader.GarbageBytes;
|
|
length = reader.Length;
|
|
}
|
|
if (length == 0 || garbage < length * CompactionRatio) continue;
|
|
compacted.Add(location.Folder);
|
|
if (dryRun) continue;
|
|
// The thumbnail builder appends under the thumbs lease only: holding it too keeps the compacted pack from
|
|
// replacing the file under a writer, whose later thumbnails would land in the orphaned one.
|
|
await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
|
|
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{location.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
|
|
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
|
|
ThumbPackWriter.Compact(sequence.ThumbsPackPath);
|
|
}
|
|
return compacted;
|
|
}
|
|
|
|
/// <summary>Sets <see cref="FrameFlags.Duplicate"/> on every copy after the first of each hash and clears it elsewhere.</summary>
|
|
private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken)
|
|
{
|
|
var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
|
|
var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId);
|
|
var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false))
|
|
.ToDictionary(f => f.FrameId, f => f.SequenceId);
|
|
var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList();
|
|
var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList();
|
|
if (dryRun) return (groups.Count, set.Count + clear.Count);
|
|
|
|
foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value))
|
|
{
|
|
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
|
|
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
|
|
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
|
|
var folder = paths.Sequence(sequence.Folder);
|
|
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
|
|
{
|
|
// The scan stamps follow only when the index held the folder as it was: frames another process listed in the
|
|
// sidecar meanwhile are still indexed by the next scan.
|
|
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
|
|
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
|
|
ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
|
|
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
|
|
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
|
|
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
|
|
{
|
|
// The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for.
|
|
if (await tx.Sequences.GetForUpdateAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not null)
|
|
{
|
|
if (setIds.Count > 0) await tx.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
|
|
if (clearIds.Count > 0) await tx.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
|
|
}
|
|
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
return (groups.Count, set.Count + clear.Count);
|
|
}
|
|
}
|