Files
Chronos/src/Chronos.Data/Archiving/ArchiveMaintenance.cs
T
Alby96andClaude Opus 5.5 f2b58522b9
CI / Build and test (push) Successful in 8m14s
CI / Windows tests and installer (push) Skipped
CI / YAML check (push) Successful in 7s
Release / Build, test and publish (push) Successful in 13m19s
CI / PostgreSQL 16 integration (push) Successful in 4m49s
CI / PostgreSQL 17 integration (push) Successful in 5m28s
CI / Docker image (push) Successful in 53s
Release / Server image (push) Failing after 10s
refactor!: rename Titano to Chronos and move CI and releases to Gitea (#6)
The owner renamed the product and moved the repository to the self-hosted
Gitea (https://gitea.encke-hake.ts.net/BalboSoft/Chronos) for more control,
and asked for Actions and a first automatic release with the server package
and the Windows exe (docs/requests/2026-10-23-chronos-gitea.md).

- Chronos everywhere: a byte-exact rename of contents and paths, then a
  per-language review of the forms the old name's ending had dictated.
- Gitea instead of GitHub: the update check, licence, issue and support
  links read ChronosReleasesUrl; the update check takes the highest stable
  version, offers a pre-release build its final release and reports a
  repository it cannot see as a failed check. EULA, security policy and
  Gitea issue forms live at the repository root.
- Gitea Actions for the Linux act_runner (.gitea/workflows): CI, release,
  nightly image and performance gates in the .NET SDK container with git
  checkout and the system SQLite. The release cross-compiles the Windows
  programs, checks their icon, version and DPI manifest, packs installers
  and zips, publishes the Gitea release and pushes the server image
  (REGISTRY_TOKEN). Windows-only tests run in an opt-in job on a Windows
  runner (WINDOWS_RUNNER).
- New Chronos.Tools commands: zip, gitea-release, postgres-ci (replaces the
  docker-exec script, which Gitea's service containers do not allow; the
  throw-away server runs without durability) and pe-check.
- Timed tests take their budget from TestTiming (CHRONOS_TEST_TIME_FACTOR,
  3 on the shared runner; the weekly performance gates keep theirs).

BREAKING CHANGE: programs, settings, data folders, file types, environment
variables and the Docker image use the new name; nothing is migrated from
Titano, which never had a release.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Alberto Balbo <alberto.balbo.96@gmail.com>
2026-10-23 12:45:18 +02:00

280 lines
16 KiB
C#

using Chronos.Core.Archiving;
using Chronos.Core.Diagnostics;
using Chronos.Core.Model;
using Chronos.Data.Repositories;
namespace Chronos.Data.Archiving;
/// <summary>Frames sharing one SHA-256; the first (earliest sequence, lowest index) counts as the original.</summary>
public sealed record DuplicateGroup(string Sha256, IReadOnlyList<DuplicateFrame> Frames);
/// <summary>Index and disk out of step (BRIEF §12.5 Orphans).</summary>
/// <param name="Sequences">Per indexed sequence: photos the sidecar does not list and frames whose file is gone.</param>
/// <param name="MissingSequences">Index rows whose folder holds no sidecar any more.</param>
/// <param name="UnindexedFolders">Folders with photos or a sidecar that the index does not know yet (a scan adopts them).</param>
public sealed record OrphanReport(IReadOnlyList<SequenceOrphans> Sequences, IReadOnlyList<SequenceLocation> MissingSequences, IReadOnlyList<string> UnindexedFolders)
{
public int UnindexedFiles => Sequences.Sum(s => s.UnindexedFiles.Count);
public int MissingFiles => Sequences.Sum(s => s.MissingFiles.Count);
}
/// <summary>What a maintenance run removed or changed (or would, in a dry run).</summary>
/// <param name="IndexReclaimable">Bytes of free pages in the SQLite index before the run; null on PostgreSQL (autovacuum) or
/// when a resumed run had passed that step.</param>
/// <param name="IndexReclaimed">Bytes the index vacuum gave back; 0 in a dry run or when too little was free to rewrite the file.</param>
public sealed record MaintenanceReport(
bool DryRun,
IReadOnlyList<string> StaleLeases,
IReadOnlyList<string> TemporaryFiles,
IReadOnlyList<string> CompactedPacks,
int DuplicateGroups,
int DuplicateFlagChanges,
int UnindexedFiles,
int MissingFiles,
long? IndexReclaimable = null,
long IndexReclaimed = 0);
/// <summary>Duplicates, orphans and housekeeping of an archive (BRIEF §12.5). Originals are never touched.</summary>
public sealed class ArchiveMaintenance
{
/// <summary>Staged copies older than this belong to no running import.</summary>
private static readonly TimeSpan StagingAge = TimeSpan.FromHours(24);
/// <summary>Temporary files of atomic writes older than this were left by a crash.</summary>
private static readonly TimeSpan TempAge = TimeSpan.FromHours(1);
/// <summary>Packs with more reclaimable bytes than this share are compacted.</summary>
private const double CompactionRatio = 0.3;
/// <summary>The SQLite index is vacuumed when at least this share of it is free pages: rewriting a large index for a
/// few pages would hold its writers for nothing.</summary>
internal const double IndexVacuumRatio = 0.1;
private readonly ChronosDatabase _db;
private readonly SequenceWriter _writer;
private readonly ArchiveIndexer _indexer;
internal ArchiveMaintenance(ChronosDatabase db, SequenceWriter writer, ArchiveIndexer indexer)
{
_db = db;
_writer = writer;
_indexer = indexer;
}
public async Task<IReadOnlyList<DuplicateGroup>> FindDuplicatesAsync(Guid archiveId, CancellationToken cancellationToken = default) =>
(await _db.Frames.ListDuplicatesAsync(archiveId, cancellationToken).ConfigureAwait(false))
.GroupBy(f => f.Sha256, StringComparer.Ordinal)
.Select(g => new DuplicateGroup(g.Key, g.ToList()))
.ToList();
public async Task<OrphanReport> FindOrphansAsync(Archive archive, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
// A sequence waiting for its first frame has no folder yet: none of it is missing, and the archive root is not its folder.
var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false);
var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList();
var sequences = new List<SequenceOrphans>();
var missing = new List<SequenceLocation>();
foreach (var location in locations)
{
cancellationToken.ThrowIfCancellationRequested();
var sequence = paths.Sequence(location.Folder);
if (!File.Exists(sequence.ManifestPath))
{
missing.Add(location);
continue;
}
var orphans = SequenceAdopter.FindOrphans(sequence, ManifestStore.ReadSequence(sequence.ManifestPath));
if (!orphans.IsEmpty) sequences.Add(orphans);
}
var known = locations.Select(l => l.Folder).ToHashSet(StringComparer.Ordinal);
var unindexed = ArchiveScanner.FindSequenceFolders(paths, cancellationToken).Select(f => f.RelativeFolder).Where(f => !known.Contains(f)).ToList();
return new OrphanReport(sequences, missing, unindexed);
}
/// <summary>
/// Drops index rows of sequences whose sidecar is gone (an orphans fix action); rows whose sidecar is back are kept, and
/// so are sequences waiting for their first frame (<see cref="ArchiveIngest.AwaitsFolder"/>), which have none yet.
/// </summary>
public async Task<int> RemoveMissingSequencesAsync(Archive archive, IReadOnlyList<Guid> sequenceIds, CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (ArchiveVolume.Probe(paths) == ArchiveAvailability.Offline) throw new ArchiveOfflineException(paths.Root, ArchiveAvailability.Offline);
var removed = 0;
foreach (var id in sequenceIds)
{
if (await _db.Sequences.GetAsync(id, cancellationToken).ConfigureAwait(false) is not { } sequence || sequence.ArchiveId != archive.Id) continue;
if (File.Exists(paths.Sequence(sequence.Folder).ManifestPath) || ArchiveIngest.AwaitsFolder(sequence, paths)) continue;
if (await _db.Sequences.DeleteAsync(id, cancellationToken).ConfigureAwait(false)) removed++;
await _db.Archives.DeleteFolderStateAsync(archive.Id, sequence.Folder, cancellationToken).ConfigureAwait(false);
}
return removed;
}
/// <summary>Number of maintenance steps (the job checkpoints after each).</summary>
internal const int StepCount = 5;
/// <summary>Runs the maintenance steps from <paramref name="firstStep"/>: stale leases, leftover temporary files,
/// thumbnail pack compaction, duplicate flags, index vacuum (SQLite). A dry run only reports.</summary>
public async Task<MaintenanceReport> RunAsync(Archive archive, bool dryRun, int firstStep, Func<int, ValueTask> stepDone,
CancellationToken cancellationToken = default)
{
var paths = new ArchivePaths(archive.RootPath);
if (!dryRun) ArchiveVolume.EnsureWritable(paths);
var all = await _db.Sequences.ListLocationsAsync(archive.Id, cancellationToken).ConfigureAwait(false);
// A sequence waiting for its first frame has no folder yet: the archive root's pack is compacted under the leases of
// the root's own sequence, which its thumbnail passes hold, never under another id's.
var awaiting = await _indexer.AwaitingFolderAsync(paths, all, cancellationToken).ConfigureAwait(false);
var locations = all.Where(l => !awaiting.Contains(l.Id)).ToList();
IReadOnlyList<string> leases = [], temps = [], packs = [];
int groups = 0, flagChanges = 0;
long? reclaimable = null;
long reclaimed = 0;
for (var step = firstStep; step < StepCount; step++)
{
cancellationToken.ThrowIfCancellationRequested();
switch (step)
{
case 0:
leases = LeaseFile.CleanupStale(paths.LocksDirectory, dryRun);
break;
case 1:
temps = PurgeTemporaryFiles(paths, locations, dryRun);
break;
case 2:
packs = await CompactPacksAsync(paths, locations, dryRun, cancellationToken).ConfigureAwait(false);
break;
case 3:
(groups, flagChanges) = await FlagDuplicatesAsync(archive, paths, dryRun, cancellationToken).ConfigureAwait(false);
break;
case 4:
(reclaimable, reclaimed) = await VacuumIndexAsync(dryRun, cancellationToken).ConfigureAwait(false);
break;
}
await stepDone(step + 1).ConfigureAwait(false);
}
var orphans = await FindOrphansAsync(archive, cancellationToken).ConfigureAwait(false);
return new MaintenanceReport(dryRun, leases, temps, packs, groups, flagChanges, orphans.UnindexedFiles, orphans.MissingFiles, reclaimable, reclaimed);
}
/// <summary>
/// The SQLite index's free pages, given back by a vacuum when they are at least <see cref="IndexVacuumRatio"/> of the
/// file (BRIEF §12.5 "SQLite vacuum"); nothing on PostgreSQL, whose autovacuum does it.
/// </summary>
private async Task<(long? Reclaimable, long Reclaimed)> VacuumIndexAsync(bool dryRun, CancellationToken cancellationToken)
{
if (await _db.SpaceAsync(cancellationToken).ConfigureAwait(false) is not { } space) return (null, 0);
if (dryRun || space.ReclaimableBytes == 0 || space.ReclaimableBytes < space.FileBytes * IndexVacuumRatio) return (space.ReclaimableBytes, 0);
var (before, after) = await _db.VacuumAsync(cancellationToken).ConfigureAwait(false);
Log.Info("archive", $"Index vacuumed: {before.FileBytes} → {after.FileBytes} bytes");
return (space.ReclaimableBytes, long.Max(0, before.FileBytes - after.FileBytes));
}
/// <summary>Staging copies and atomic-write temporaries left behind by crashes (never originals).</summary>
private static List<string> PurgeTemporaryFiles(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun)
{
var now = DateTime.UtcNow;
var found = new List<string>();
void Collect(string directory, string pattern, TimeSpan age, bool recursive)
{
if (!Directory.Exists(directory)) return;
foreach (var file in Directory.EnumerateFiles(directory, pattern, recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly))
{
if (now - File.GetLastWriteTimeUtc(file) > age) found.Add(file);
}
}
Collect(paths.StagingDirectory, "*", StagingAge, recursive: true);
Collect(paths.MetaDirectory, ".*.tmp", TempAge, recursive: false);
foreach (var location in locations)
{
var sequence = paths.Sequence(location.Folder);
Collect(sequence.MetaDirectory, ".*.tmp", TempAge, recursive: false);
Collect(sequence.Proxies, ".*.tmp", TempAge, recursive: false);
}
if (!dryRun)
{
foreach (var file in found)
{
try
{
File.Delete(file);
}
catch (IOException ex)
{
Log.Warning("archive", $"Could not remove {file}", ex);
}
}
}
return found.ConvertAll(paths.ToRelative);
}
private async Task<List<string>> CompactPacksAsync(ArchivePaths paths, IReadOnlyList<SequenceLocation> locations, bool dryRun, CancellationToken cancellationToken)
{
var compacted = new List<string>();
foreach (var location in locations)
{
var sequence = paths.Sequence(location.Folder);
long garbage, length;
using (var reader = ThumbPackReader.Open(sequence.ThumbsPackPath))
{
garbage = reader.GarbageBytes;
length = reader.Length;
}
if (length == 0 || garbage < length * CompactionRatio) continue;
compacted.Add(location.Folder);
if (dryRun) continue;
// The thumbnail builder appends under the thumbs lease only: holding it too keeps the compacted pack from
// replacing the file under a writer, whose later thumbnails would land in the orphaned one.
await using (await _writer.LockAsync(paths, location.Id, cancellationToken).ConfigureAwait(false))
using (await LeaseFile.AcquireAsync(paths.LocksDirectory, $"thumbs-{location.Id:N}", _writer.Options.Owner, _writer.Options.LeaseWait,
_writer.Options.LeaseDuration, cancellationToken).ConfigureAwait(false))
ThumbPackWriter.Compact(sequence.ThumbsPackPath);
}
return compacted;
}
/// <summary>Sets <see cref="FrameFlags.Duplicate"/> on every copy after the first of each hash and clears it elsewhere.</summary>
private async Task<(int Groups, int Changes)> FlagDuplicatesAsync(Archive archive, ArchivePaths paths, bool dryRun, CancellationToken cancellationToken)
{
var groups = await FindDuplicatesAsync(archive.Id, cancellationToken).ConfigureAwait(false);
var wanted = groups.SelectMany(g => g.Frames.Skip(1)).ToDictionary(f => f.FrameId, f => f.SequenceId);
var flagged = (await _db.Frames.ListFlaggedAsync(archive.Id, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false))
.ToDictionary(f => f.FrameId, f => f.SequenceId);
var set = wanted.Where(w => !flagged.ContainsKey(w.Key)).ToList();
var clear = flagged.Where(f => !wanted.ContainsKey(f.Key)).ToList();
if (dryRun) return (groups.Count, set.Count + clear.Count);
foreach (var bySequence in set.Select(s => (s.Key, s.Value, Set: true)).Concat(clear.Select(c => (c.Key, c.Value, Set: false))).GroupBy(x => x.Value))
{
if (await _db.Sequences.GetAsync(bySequence.Key, cancellationToken).ConfigureAwait(false) is not { } sequence) continue;
var changes = bySequence.ToDictionary(x => x.Key, x => (Func<SidecarFrame, SidecarFrame>)(f =>
f with { Flags = x.Set ? f.Flags | FrameFlags.Duplicate : f.Flags & ~FrameFlags.Duplicate }));
var folder = paths.Sequence(sequence.Folder);
await using (await _writer.LockAsync(paths, sequence.Id, cancellationToken).ConfigureAwait(false))
{
// The scan stamps follow only when the index held the folder as it was: frames another process listed in the
// sidecar meanwhile are still indexed by the next scan.
var manifest = SequenceWriter.ReadLocked(folder, sequence.Id);
var inStep = await _indexer.InStepAsync(archive, folder, manifest, cancellationToken).ConfigureAwait(false);
ManifestStore.WriteSequence(folder.ManifestPath, SequenceFrames.Update(manifest, changes));
var setIds = bySequence.Where(x => x.Set).Select(x => x.Key).ToList();
var clearIds = bySequence.Where(x => !x.Set).Select(x => x.Key).ToList();
await using (var tx = await _db.BeginAsync(cancellationToken).ConfigureAwait(false))
{
// The row before its frames (SequenceRepository.GetForUpdateAsync), so a scan writing them waits or is waited for.
if (await tx.Sequences.GetForUpdateAsync(sequence.Id, cancellationToken).ConfigureAwait(false) is not null)
{
if (setIds.Count > 0) await tx.Frames.UpdateFlagsAsync(setIds, FrameFlags.Duplicate, FrameFlags.None, cancellationToken).ConfigureAwait(false);
if (clearIds.Count > 0) await tx.Frames.UpdateFlagsAsync(clearIds, FrameFlags.None, FrameFlags.Duplicate, cancellationToken).ConfigureAwait(false);
}
await tx.CommitAsync(cancellationToken).ConfigureAwait(false);
}
if (inStep) await _indexer.RecordStampsAsync(archive, folder, sequence.Id, cancellationToken).ConfigureAwait(false);
}
}
return (groups.Count, set.Count + clear.Count);
}
}