Files
Khaemhat/tools/Khaemhat.PlacesTool/Program.cs
T
Alby96andClaude Opus 5.5 a9eda510f2 feat: postal codes from GeoNames for the ISTAT municipalities (#45)
The municipalities now come with their postal codes, so the lookup from
postal code works on a new company without loading anything. GeoNames
(CC BY 4.0) lists places rather than municipalities: a postal code goes
to a municipality only when the name and the province abbreviation match
exactly one of them (the name alone within Sardinia, whose abbreviations
changed), which covers 7,750 of 7,894 municipalities; the rest are mostly
renamed or merged ones that users complete by hand.

tools/Khaemhat.PlacesTool rebuilds both lists from the ISTAT sheet and
the GeoNames file, framework only, so the yearly update is repeatable.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 09:16:07 +01:00

206 lines
7.8 KiB
C#

// Khaemhat.PlacesTool: rebuilds Data/province.csv and Data/comuni.csv of Khaemhat.Fiscal.Italy (ADR 0028) from the
// ISTAT list of municipality codes (xlsx, CC BY 4.0) and, optionally, the GeoNames postal codes of Italy (IT.zip,
// CC BY 4.0). The downloads stay outside the repository; only the two files it writes are committed.
using System.Globalization;
using System.IO.Compression;
using System.Text;
using System.Xml.Linq;
const string Usage = """
Khaemhat.PlacesTool
--istat <Elenco-comuni-italiani.xlsx> [--postal <IT.zip>] --out <directory>
Writes province.csv and comuni.csv. Postal codes are matched to the municipalities by name and province
abbreviation (by name alone in Sardinia, whose abbreviations changed); hamlets are ignored.
""";
var options = new Dictionary<string, string>(StringComparer.Ordinal);
for (var i = 0; i + 1 < args.Length; i += 2)
{
options[args[i]] = args[i + 1];
}
if (!options.TryGetValue("--istat", out var istatPath) || !options.TryGetValue("--out", out var outDir))
{
Console.WriteLine(Usage);
return 1;
}
var sheet = ReadFirstSheet(istatPath);
var provinces = new SortedDictionary<string, (string Name, string Region)>(StringComparer.Ordinal);
var municipalities = new List<Place>();
foreach (var row in sheet.Skip(1).Where(r => r.Count > 20 && r[20].Trim().Length > 0))
{
var place = new Place(row[20].Trim().ToUpperInvariant(), row[6].Trim(), row[14].Trim(), row[10].Trim(), row[4].Trim());
if (place.Code.Length != 4 || place.Province.Length != 2 || place.Istat.Length != 6 || place.Name.Contains(';', StringComparison.Ordinal))
{
throw new InvalidDataException($"Unexpected row: {string.Join('|', row)}");
}
var province = (row[11].Trim(), place.Region);
if (provinces.TryGetValue(place.Province, out var known) && known != province)
{
throw new InvalidDataException($"Province {place.Province} has two names: {known} and {province}.");
}
provinces[place.Province] = province;
municipalities.Add(place);
}
if (municipalities.Select(m => m.Code).Distinct(StringComparer.Ordinal).Count() != municipalities.Count)
{
throw new InvalidDataException("Duplicate cadastral codes.");
}
var postal = options.TryGetValue("--postal", out var postalPath) ? MatchPostalCodes(postalPath, municipalities) : [];
Directory.CreateDirectory(outDir);
var provinceText = new StringBuilder("sigla;denominazione;regione\n");
foreach (var (code, (name, region)) in provinces)
{
provinceText.Append(CultureInfo.InvariantCulture, $"{code};{name};{region}\n");
}
var municipalityText = new StringBuilder("codice_catastale;denominazione;sigla;regione;codice_istat;cap\n");
foreach (var m in municipalities.OrderBy(m => m.Istat, StringComparer.Ordinal))
{
var codes = postal.TryGetValue(m.Code, out var set) ? Ranges(set) : string.Empty;
municipalityText.Append(CultureInfo.InvariantCulture, $"{m.Code};{m.Name};{m.Province};{m.Region};{m.Istat};{codes}\n");
}
await File.WriteAllTextAsync(Path.Combine(outDir, "province.csv"), provinceText.ToString(), new UTF8Encoding(false));
await File.WriteAllTextAsync(Path.Combine(outDir, "comuni.csv"), municipalityText.ToString(), new UTF8Encoding(false));
Console.WriteLine($"{provinces.Count} provinces, {municipalities.Count} municipalities, {postal.Count} with postal codes.");
return 0;
// The rows of the first sheet of an xlsx, read with the framework only (shared strings and inline strings).
static List<List<string>> ReadFirstSheet(string path)
{
XNamespace ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
using var zip = ZipFile.OpenRead(path);
var shared = zip.GetEntry("xl/sharedStrings.xml") is { } entry
? [.. Load(entry).Root!.Elements(ns + "si").Select(si => string.Concat(si.Descendants(ns + "t").Select(t => t.Value)))]
: new List<string>();
var sheetEntry = zip.Entries.Where(e => e.FullName.StartsWith("xl/worksheets/sheet", StringComparison.Ordinal)).OrderBy(e => e.FullName, StringComparer.Ordinal).First();
var rows = new List<List<string>>();
foreach (var row in Load(sheetEntry).Descendants(ns + "row"))
{
var cells = new SortedDictionary<int, string>();
foreach (var cell in row.Elements(ns + "c"))
{
var value = cell.Element(ns + "v")?.Value;
cells[Column((string)cell.Attribute("r")!)] = (string?)cell.Attribute("t") switch
{
"s" when value is not null => shared[int.Parse(value, CultureInfo.InvariantCulture)],
"inlineStr" => string.Concat(cell.Descendants(ns + "t").Select(t => t.Value)),
_ => value ?? string.Empty,
};
}
var width = cells.Count == 0 ? 0 : cells.Keys.Max() + 1;
rows.Add([.. Enumerable.Range(0, width).Select(i => cells.GetValueOrDefault(i, string.Empty))]);
}
return rows;
static XDocument Load(ZipArchiveEntry entry)
{
using var stream = entry.Open();
return XDocument.Load(stream);
}
static int Column(string reference)
{
var n = 0;
foreach (var ch in reference.TakeWhile(char.IsAsciiLetterUpper))
{
n = (n * 26) + (ch - 'A' + 1);
}
return n - 1;
}
}
// GeoNames rows: country, postal code, place, region, region code, province, abbreviation, ...
static Dictionary<string, SortedSet<string>> MatchPostalCodes(string path, List<Place> municipalities)
{
using var zip = ZipFile.OpenRead(path);
using var reader = new StreamReader(zip.GetEntry("IT.txt")!.Open(), Encoding.UTF8);
var byNameAndProvince = municipalities.ToLookup(m => (Normalize(m.Name), m.Province));
var byName = municipalities.ToLookup(m => Normalize(m.Name));
var result = new Dictionary<string, SortedSet<string>>(StringComparer.Ordinal);
while (reader.ReadLine() is { } line)
{
var fields = line.Split('\t');
var name = Normalize(fields[2]);
var match = byNameAndProvince[(name, fields[6])].ToList();
if (match.Count != 1 && fields[3] == "Sardegna")
{
match = [.. byName[name].Where(m => m.Region == "Sardegna")];
}
if (match.Count == 1)
{
if (!result.TryGetValue(match[0].Code, out var set))
{
result[match[0].Code] = set = new SortedSet<string>(StringComparer.Ordinal);
}
set.Add(fields[1]);
}
}
return result;
}
// Lower case without accents, apostrophes or hyphens; the Italian part of bilingual names.
static string Normalize(string name)
{
var text = name.Split('/')[0].Normalize(NormalizationForm.FormD);
var builder = new StringBuilder();
foreach (var ch in text)
{
if (CharUnicodeInfo.GetUnicodeCategory(ch) == UnicodeCategory.NonSpacingMark)
{
continue;
}
builder.Append(ch is '-' or '\'' or '`' or '.' ? ' ' : char.ToLowerInvariant(ch));
}
return string.Join(' ', builder.ToString().Split(' ', StringSplitOptions.RemoveEmptyEntries));
}
// 10121,10122,10123 becomes 10121-10123.
static string Ranges(SortedSet<string> codes)
{
var parts = new List<string>();
string? start = null;
string? previous = null;
foreach (var code in codes)
{
if (start is not null && int.Parse(code, CultureInfo.InvariantCulture) == int.Parse(previous!, CultureInfo.InvariantCulture) + 1)
{
previous = code;
continue;
}
if (start is not null)
{
parts.Add(start == previous ? start : $"{start}-{previous}");
}
start = previous = code;
}
if (start is not null)
{
parts.Add(start == previous ? start : $"{start}-{previous}");
}
return string.Join(',', parts);
}
internal sealed record Place(string Code, string Name, string Province, string Region, string Istat);