The render upload refusal render_exists had no Server.Error.RenderExists resource, so clients showed the raw code. ApiErrors builds that key at run time, which the scan of used keys cannot see: i18n check now maps every snake_case code literal the server passes to ApiErrors, Outcome.Fail and its Fail and TooMany helpers to its Server.Error.* key and fails when one is missing (SRV-6). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012iLMBNwBE2GnphAsu4cjaC
364 lines
20 KiB
C#
364 lines
20 KiB
C#
using System.Text;
|
||
using System.Text.Json;
|
||
using System.Text.RegularExpressions;
|
||
using System.Xml.Linq;
|
||
using Titano.Core.Localization;
|
||
|
||
namespace Titano.Tools.Commands;
|
||
|
||
/// <summary>
|
||
/// <c>i18n check</c>: completeness of <c>en</c> and <c>it</c>, placeholder parity, length budgets, that every key
|
||
/// used in code/XAML/admin JS exists and that every error code the server answers with has its message (BRIEF §16).
|
||
/// <c>i18n export --out dir</c>: one JSON per culture for the web admin.
|
||
/// <c>i18n missing --culture xx [--all]</c>: untranslated (or all) keys as TSV for a translator or reviewer.
|
||
/// <c>i18n import --culture xx file.tsv</c>: merges translated <c>key<TAB>text</c> rows into the culture's resx files.
|
||
/// </summary>
|
||
internal sealed partial class I18nCommand : ICommand
|
||
{
|
||
private static readonly string[] RequiredCultures = ["en", "it"];
|
||
private static readonly HashSet<string> SentenceAreas = new(StringComparer.Ordinal) { "Tooltips", "Errors", "Advisor" };
|
||
|
||
public string Name => "i18n";
|
||
public string Summary => "check | export --out <dir> | missing --culture <c> [--all] | import --culture <c> <tsv>";
|
||
|
||
public Task<int> RunAsync(string[] args)
|
||
{
|
||
var a = new Args(args, "all");
|
||
var sub = a.Positional.FirstOrDefault() ?? "check";
|
||
var root = Args.RepoRoot();
|
||
var catalog = Catalog.Load(Path.Combine(root, "i18n"));
|
||
return Task.FromResult(sub switch
|
||
{
|
||
"check" => Check(root, catalog),
|
||
"export" => Export(catalog, a.Get("out") ?? throw new ToolException("--out <dir> required")),
|
||
"missing" => Missing(catalog, a.Get("culture") ?? throw new ToolException("--culture <code> required"), a.Has("all")),
|
||
"import" => Import(root, catalog, a.Get("culture") ?? throw new ToolException("--culture <code> required"),
|
||
a.Positional.ElementAtOrDefault(1) ?? throw new ToolException("TSV file required")),
|
||
_ => throw new ToolException($"Unknown i18n subcommand '{sub}'"),
|
||
});
|
||
}
|
||
|
||
private static int Check(string root, Catalog catalog)
|
||
{
|
||
var errors = new List<string>();
|
||
var warnings = new List<string>();
|
||
if (!catalog.Cultures.TryGetValue("en", out var en)) throw new ToolException("i18n/en missing");
|
||
|
||
foreach (var (key, entry) in en)
|
||
{
|
||
var area = key.Split('.')[0];
|
||
if (!entry.File.Equals(area, StringComparison.Ordinal))
|
||
errors.Add($"en: key {key} lives in {entry.File}.resx but its area is {area}");
|
||
var limit = Limit(key, area, entry.Comment);
|
||
if (entry.Value.Length > limit)
|
||
errors.Add($"en: {key} is {entry.Value.Length} chars (budget {limit})");
|
||
if (string.IsNullOrWhiteSpace(entry.Value))
|
||
errors.Add($"en: {key} is empty");
|
||
}
|
||
|
||
foreach (var (culture, strings) in catalog.Cultures)
|
||
{
|
||
if (culture == "en") continue;
|
||
var required = RequiredCultures.Contains(culture);
|
||
var missing = 0;
|
||
foreach (var (key, entry) in en)
|
||
{
|
||
if (entry.Comment is not null && entry.Comment.Contains("invariant", StringComparison.OrdinalIgnoreCase)) continue;
|
||
if (!strings.TryGetValue(key, out var translated))
|
||
{
|
||
if (required) errors.Add($"{culture}: missing {key}");
|
||
else missing++;
|
||
continue;
|
||
}
|
||
CheckTranslation(culture, key, entry, translated, en, errors, warnings);
|
||
}
|
||
if (missing > 0) warnings.Add($"{culture}: {missing} key(s) not translated yet");
|
||
foreach (var (key, translated) in strings.Where(s => !en.ContainsKey(s.Key)))
|
||
{
|
||
// Languages with more plural forms than English add key.few / key.many next to key.one / key.other.
|
||
if (PluralStem(key, en) is { } other) CheckTranslation(culture, key, other, translated, en, errors, warnings);
|
||
else errors.Add($"{culture}: orphan key {key}");
|
||
}
|
||
}
|
||
|
||
var areas = en.Keys.Select(k => k[..k.IndexOf('.')]).ToHashSet(StringComparer.Ordinal);
|
||
foreach (var (file, key) in UsedKeys(root, areas))
|
||
{
|
||
if (en.ContainsKey(key)) continue;
|
||
// Prefix lookups such as L("Settings.Page." + page) are satisfied by any key under the prefix.
|
||
if (key.EndsWith('.') && en.Keys.Any(k => k.StartsWith(key, StringComparison.Ordinal))) continue;
|
||
// Plural lookups reference the stem; any suffix satisfies them.
|
||
if (en.Keys.Any(k => k.StartsWith(key + ".", StringComparison.Ordinal) && PluralSuffix().IsMatch(k[(key.Length + 1)..]))) continue;
|
||
errors.Add($"{Path.GetRelativePath(root, file)}: unknown key {key}");
|
||
}
|
||
foreach (var (file, code, key) in MissingErrorMessages(root, en.Keys.ToHashSet(StringComparer.Ordinal)))
|
||
errors.Add($"{Path.GetRelativePath(root, file)}: error code {code} has no message {key}");
|
||
|
||
foreach (var w in warnings.Take(50)) Console.WriteLine("warning: " + w);
|
||
if (warnings.Count > 50) Console.WriteLine($"warning: … {warnings.Count - 50} more");
|
||
foreach (var e in errors) Console.Error.WriteLine("error: " + e);
|
||
Console.WriteLine($"i18n: {en.Count} keys, {catalog.Cultures.Count} cultures, {errors.Count} error(s), {warnings.Count} warning(s)");
|
||
return errors.Count == 0 ? 0 : 1;
|
||
}
|
||
|
||
/// <summary>A translation keeps the English placeholders and stays within 1.6× the English budget. Plural forms
|
||
/// are compared with the English <c>other</c> form and may drop the count <c>{0}</c> (as English "Rendered once" does).</summary>
|
||
private static void CheckTranslation(string culture, string key, Entry source, Entry translated, Dictionary<string, Entry> en, List<string> errors, List<string> warnings)
|
||
{
|
||
var p2 = Placeholders(translated.Value);
|
||
if (PluralStem(key, en) is { } other)
|
||
{
|
||
var p1 = Placeholders(other.Value);
|
||
p1.Remove("0");
|
||
if (!p2.IsSubsetOf([.. p1, "0"]) || !p1.IsSubsetOf(p2))
|
||
errors.Add($"{culture}: {key} placeholders {{{string.Join(",", p2)}}} differ from en {{{string.Join(",", p1.Prepend("0"))}}}");
|
||
}
|
||
else
|
||
{
|
||
var p1 = Placeholders(source.Value);
|
||
if (!p1.SetEquals(p2))
|
||
errors.Add($"{culture}: {key} placeholders {{{string.Join(",", p2)}}} differ from en {{{string.Join(",", p1)}}}");
|
||
}
|
||
if (string.IsNullOrWhiteSpace(translated.Value))
|
||
errors.Add($"{culture}: {key} is empty");
|
||
var limit = (int)(Limit(key, key.Split('.')[0], source.Comment) * 1.6);
|
||
if (translated.Value.Length > limit)
|
||
warnings.Add($"{culture}: {key} is {translated.Value.Length} chars (soft budget {limit})");
|
||
}
|
||
|
||
/// <summary>The English <c>other</c> form when <paramref name="key"/> is a plural form of an English plural key.</summary>
|
||
private static Entry? PluralStem(string key, Dictionary<string, Entry> en)
|
||
{
|
||
var dot = key.LastIndexOf('.');
|
||
if (dot <= 0 || !PluralSuffix().IsMatch(key[(dot + 1)..])) return null;
|
||
return en.GetValueOrDefault(key[..dot] + ".other");
|
||
}
|
||
|
||
/// <summary>Untranslated keys as <c>key, English, budget, note</c>; with <c>--all</c> every key plus its current
|
||
/// translation (plural forms the language adds included) for review.</summary>
|
||
private static int Missing(Catalog catalog, string culture, bool all)
|
||
{
|
||
var en = catalog.Cultures["en"];
|
||
catalog.Cultures.TryGetValue(culture, out var target);
|
||
target ??= [];
|
||
var rows = en.Select(e => (Key: e.Key, Source: e.Value))
|
||
.Concat(target.Keys.Where(k => !en.ContainsKey(k)).Select(k => (Key: k, Source: PluralStem(k, en)!)))
|
||
.Where(r => r.Source is not null)
|
||
.OrderBy(r => r.Key, StringComparer.Ordinal);
|
||
foreach (var (key, source) in rows)
|
||
{
|
||
if (source.Comment is not null && source.Comment.Contains("invariant", StringComparison.OrdinalIgnoreCase)) continue;
|
||
var translated = target.GetValueOrDefault(key);
|
||
if (translated is not null && !all) continue;
|
||
var budget = (int)(Limit(key, key.Split('.')[0], source.Comment) * 1.6);
|
||
Console.WriteLine(all ? $"{key}\t{source.Value}\t{budget}\t{source.Comment}\t{translated?.Value}" : $"{key}\t{source.Value}\t{budget}\t{source.Comment}");
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/// <summary>
|
||
/// Merges <c>key<TAB>text</c> rows (further columns and <c>#</c> lines ignored) into
|
||
/// <c>i18n/<culture>/<Area>.<culture>.resx</c>, replacing existing values and keeping keys sorted.
|
||
/// Keys must exist in English or be a plural form of an English plural key; placeholders are verified by <c>check</c>.
|
||
/// </summary>
|
||
private static int Import(string root, Catalog catalog, string culture, string tsv)
|
||
{
|
||
if (culture.Equals("en", StringComparison.OrdinalIgnoreCase)) throw new ToolException("English is the source; edit i18n/en directly");
|
||
if (!Languages.Shipped.Contains(culture, StringComparer.OrdinalIgnoreCase)) throw new ToolException($"{culture} is not a shipped language");
|
||
culture = Languages.Shipped.First(c => c.Equals(culture, StringComparison.OrdinalIgnoreCase));
|
||
var en = catalog.Cultures["en"];
|
||
var rows = new Dictionary<string, string>(StringComparer.Ordinal);
|
||
var line = 0;
|
||
foreach (var raw in File.ReadLines(tsv))
|
||
{
|
||
line++;
|
||
if (raw.Length == 0 || raw.StartsWith('#')) continue;
|
||
var cells = raw.Split('\t');
|
||
if (cells.Length < 2) throw new ToolException($"{tsv}:{line}: expected key<TAB>text");
|
||
var key = cells[0].Trim();
|
||
if (!en.ContainsKey(key) && PluralStem(key, en) is null) throw new ToolException($"{tsv}:{line}: unknown key {key}");
|
||
if (!rows.TryAdd(key, cells[1])) throw new ToolException($"{tsv}:{line}: duplicate key {key}");
|
||
}
|
||
|
||
var dir = Path.Combine(root, "i18n", culture);
|
||
Directory.CreateDirectory(dir);
|
||
var written = 0;
|
||
foreach (var group in rows.GroupBy(r => r.Key[..r.Key.IndexOf('.')], StringComparer.Ordinal))
|
||
{
|
||
var path = Path.Combine(dir, $"{group.Key}.{culture}.resx");
|
||
var values = new SortedDictionary<string, string>(StringComparer.Ordinal);
|
||
if (File.Exists(path))
|
||
{
|
||
foreach (var data in XDocument.Load(path).Root!.Elements("data"))
|
||
values[(string)data.Attribute("name")!] = (string?)data.Element("value") ?? string.Empty;
|
||
}
|
||
foreach (var (key, value) in group) values[key] = value;
|
||
WriteResx(path, values);
|
||
written += group.Count();
|
||
}
|
||
Console.WriteLine($"{culture}: {written} string(s) imported");
|
||
return 0;
|
||
}
|
||
|
||
private static void WriteResx(string path, IEnumerable<KeyValuePair<string, string>> values)
|
||
{
|
||
var sb = new StringBuilder();
|
||
sb.Append("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<root>\n");
|
||
sb.Append(" <resheader name=\"resmimetype\"><value>text/microsoft-resx</value></resheader>\n");
|
||
sb.Append(" <resheader name=\"version\"><value>2.0</value></resheader>\n");
|
||
sb.Append(" <resheader name=\"reader\"><value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>\n");
|
||
sb.Append(" <resheader name=\"writer\"><value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value></resheader>\n");
|
||
foreach (var (key, value) in values)
|
||
sb.Append($" <data name=\"{Escape(key)}\" xml:space=\"preserve\"><value>{Escape(value)}</value></data>\n");
|
||
sb.Append("</root>\n");
|
||
// Atomic replace: other translators may be running `i18n check` over the same tree.
|
||
var temp = path + ".tmp";
|
||
File.WriteAllText(temp, sb.ToString(), new UTF8Encoding(false));
|
||
File.Move(temp, path, overwrite: true);
|
||
}
|
||
|
||
private static string Escape(string s) => s.Replace("&", "&", StringComparison.Ordinal).Replace("<", "<", StringComparison.Ordinal)
|
||
.Replace(">", ">", StringComparison.Ordinal).Replace("\"", """, StringComparison.Ordinal);
|
||
|
||
private static int Limit(string key, string area, string? comment)
|
||
{
|
||
if (comment is not null && comment.Contains("long", StringComparison.OrdinalIgnoreCase)) return 200;
|
||
if (SentenceAreas.Contains(area) || key.EndsWith(".Tip", StringComparison.Ordinal)) return 200;
|
||
return 30;
|
||
}
|
||
|
||
private static int Export(Catalog catalog, string outDir)
|
||
{
|
||
Directory.CreateDirectory(outDir);
|
||
var en = catalog.Cultures["en"];
|
||
foreach (var (culture, strings) in catalog.Cultures)
|
||
{
|
||
var merged = new SortedDictionary<string, string>(StringComparer.Ordinal);
|
||
foreach (var (k, v) in en) merged[k] = strings.TryGetValue(k, out var t) ? t.Value : v.Value;
|
||
var json = JsonSerializer.Serialize(merged, new JsonSerializerOptions { WriteIndented = true, Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
|
||
File.WriteAllText(Path.Combine(outDir, culture + ".json"), json, new UTF8Encoding(false));
|
||
}
|
||
Console.WriteLine($"Exported {catalog.Cultures.Count} cultures to {outDir}");
|
||
return 0;
|
||
}
|
||
|
||
private static HashSet<string> Placeholders(string s) =>
|
||
[.. PlaceholderRegex().Matches(s).Select(m => m.Groups[1].Value)];
|
||
|
||
/// <summary>
|
||
/// Keys referenced from C# (<c>L("…")</c>, <c>Plural("…")</c>, <c>Loc.Get("…")</c>, and any string literal shaped
|
||
/// <c>Area.Group.Name</c> whose area has a catalog, since reason and warning keys travel as data), XAML
|
||
/// (<c>{l:Loc …}</c>) and admin JS (<c>t("…")</c>, <c>plural("…")</c>).
|
||
/// </summary>
|
||
private static IEnumerable<(string File, string Key)> UsedKeys(string root, IReadOnlySet<string> areas)
|
||
{
|
||
var src = Path.Combine(root, "src");
|
||
foreach (var file in Directory.EnumerateFiles(src, "*.*", SearchOption.AllDirectories))
|
||
{
|
||
if (IsBuildOutput(file)) continue;
|
||
var ext = Path.GetExtension(file);
|
||
Regex? regex = ext switch
|
||
{
|
||
".cs" => CsKeyRegex(),
|
||
".xaml" => XamlKeyRegex(),
|
||
".js" or ".html" => JsKeyRegex(),
|
||
_ => null,
|
||
};
|
||
if (regex is null) continue;
|
||
var text = File.ReadAllText(file);
|
||
foreach (Match m in regex.Matches(text)) yield return (file, m.Groups[1].Value);
|
||
if (ext != ".cs") continue;
|
||
foreach (Match m in CsKeyLiteralRegex().Matches(text))
|
||
{
|
||
if (areas.Contains(m.Groups[2].Value)) yield return (file, m.Groups[1].Value);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Error codes of the server (literals passed to <c>ApiErrors</c>, <c>Outcome.Fail</c> and the local <c>Fail</c> and
|
||
/// <c>TooMany</c> helpers) whose message <c>Server.Error.<Code></c> is not in <paramref name="keys"/>, each once.
|
||
/// <c>ApiErrors.Message</c> builds that key at run time, so the scan of used keys cannot see it.
|
||
/// </summary>
|
||
internal static IEnumerable<(string File, string Code, string Key)> MissingErrorMessages(string root, IReadOnlySet<string> keys)
|
||
{
|
||
var reported = new HashSet<string>(StringComparer.Ordinal);
|
||
var server = Path.Combine(root, "src", "Titano.Server");
|
||
foreach (var file in Directory.EnumerateFiles(server, "*.cs", SearchOption.AllDirectories).Order(StringComparer.Ordinal))
|
||
{
|
||
if (IsBuildOutput(file)) continue;
|
||
foreach (var code in ErrorCodes(File.ReadAllText(file)))
|
||
{
|
||
var key = ErrorKey(code);
|
||
if (!keys.Contains(key) && reported.Add(code)) yield return (file, code, key);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>The literal error codes (snake_case) a C# source passes to the server's error helpers.</summary>
|
||
internal static IEnumerable<string> ErrorCodes(string source) => ErrorCodeRegex().Matches(source).Select(m => m.Groups[1].Value);
|
||
|
||
/// <summary>The message key of an error code, built like <c>ApiErrors.Message</c>: <c>render_exists</c> → <c>Server.Error.RenderExists</c>.</summary>
|
||
internal static string ErrorKey(string code) =>
|
||
"Server.Error." + string.Concat(code.Split('_', StringSplitOptions.RemoveEmptyEntries).Select(part => char.ToUpperInvariant(part[0]) + part[1..]));
|
||
|
||
private static bool IsBuildOutput(string file) =>
|
||
file.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", StringComparison.Ordinal)
|
||
|| file.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}", StringComparison.Ordinal);
|
||
|
||
[GeneratedRegex(@"\{(\d+)(?:[,:][^}]*)?\}")]
|
||
private static partial Regex PlaceholderRegex();
|
||
|
||
/// <summary>A call of an error helper up to its first string argument (no statement end in between) when that is a snake_case code.</summary>
|
||
[GeneratedRegex(@"(?:\bApiErrors\.(?:Result|WriteAsync|BadRequest|Conflict|Message)|\bFail(?:<[^;()]*?>)?|\bTooMany)\([^;""]*?""([a-z][a-z0-9_]*)""")]
|
||
private static partial Regex ErrorCodeRegex();
|
||
|
||
[GeneratedRegex(@"\b(?:L|Plural|Loc\.Get|LocKey)\(\s*""([A-Z][A-Za-z0-9]*\.[A-Za-z0-9_.]+)""")]
|
||
private static partial Regex CsKeyRegex();
|
||
|
||
[GeneratedRegex(@"(?<![@$\w])""(([A-Z][A-Za-z0-9]*)\.[A-Z][A-Za-z0-9]*\.[A-Za-z0-9_.]+)""")]
|
||
private static partial Regex CsKeyLiteralRegex();
|
||
|
||
[GeneratedRegex(@"\{(?:l|loc):Loc\s+(?:Key=)?([A-Z][A-Za-z0-9]*\.[A-Za-z0-9_.]+)")]
|
||
private static partial Regex XamlKeyRegex();
|
||
|
||
[GeneratedRegex(@"(?:\b(?:t|plural)\(\s*['""]|data-i18n(?:-[a-z]+)?=['""])([A-Z][A-Za-z0-9]*\.[A-Za-z0-9_.]+)")]
|
||
private static partial Regex JsKeyRegex();
|
||
|
||
[GeneratedRegex(@"^(one|few|many|other)$")]
|
||
private static partial Regex PluralSuffix();
|
||
|
||
internal sealed record Entry(string File, string Value, string? Comment);
|
||
|
||
internal sealed class Catalog
|
||
{
|
||
public Dictionary<string, Dictionary<string, Entry>> Cultures { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||
|
||
public static Catalog Load(string dir)
|
||
{
|
||
var c = new Catalog();
|
||
foreach (var cultureDir in Directory.EnumerateDirectories(dir))
|
||
{
|
||
var culture = Path.GetFileName(cultureDir);
|
||
var strings = new Dictionary<string, Entry>(StringComparer.Ordinal);
|
||
foreach (var file in Directory.EnumerateFiles(cultureDir, "*.resx"))
|
||
{
|
||
var area = Path.GetFileName(file).Split('.')[0];
|
||
var doc = XDocument.Load(file);
|
||
foreach (var data in doc.Root!.Elements("data"))
|
||
{
|
||
var name = (string?)data.Attribute("name") ?? throw new ToolException($"{file}: data without name");
|
||
var value = (string?)data.Element("value") ?? string.Empty;
|
||
if (!strings.TryAdd(name, new Entry(area, value, (string?)data.Element("comment"))))
|
||
throw new ToolException($"{file}: duplicate key {name}");
|
||
}
|
||
}
|
||
c.Cultures[culture] = strings;
|
||
}
|
||
foreach (var code in Languages.Shipped.Where(s => !c.Cultures.ContainsKey(s)))
|
||
c.Cultures.TryAdd(code, new Dictionary<string, Entry>(StringComparer.Ordinal));
|
||
return c;
|
||
}
|
||
}
|
||
}
|