The municipalities now come with their postal codes, so the lookup from postal code works on a new company without loading anything. GeoNames (CC BY 4.0) lists places rather than municipalities: a postal code goes to a municipality only when the name and the province abbreviation match exactly one of them (the name alone within Sardinia, whose abbreviations changed), which covers 7,750 of 7,894 municipalities; the rest are mostly renamed or merged ones that users complete by hand. tools/Khaemhat.PlacesTool rebuilds both lists from the ISTAT sheet and the GeoNames file, framework only, so the yearly update is repeatable. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
206 lines
7.8 KiB
C#
206 lines
7.8 KiB
C#
// Khaemhat.PlacesTool: rebuilds Data/province.csv and Data/comuni.csv of Khaemhat.Fiscal.Italy (ADR 0028) from the
|
|
// ISTAT list of municipality codes (xlsx, CC BY 4.0) and, optionally, the GeoNames postal codes of Italy (IT.zip,
|
|
// CC BY 4.0). The downloads stay outside the repository; only the two files it writes are committed.
|
|
|
|
using System.Globalization;
|
|
using System.IO.Compression;
|
|
using System.Text;
|
|
using System.Xml.Linq;
|
|
|
|
const string Usage = """
|
|
Khaemhat.PlacesTool
|
|
|
|
--istat <Elenco-comuni-italiani.xlsx> [--postal <IT.zip>] --out <directory>
|
|
Writes province.csv and comuni.csv. Postal codes are matched to the municipalities by name and province
|
|
abbreviation (by name alone in Sardinia, whose abbreviations changed); hamlets are ignored.
|
|
""";
|
|
|
|
var options = new Dictionary<string, string>(StringComparer.Ordinal);
|
|
for (var i = 0; i + 1 < args.Length; i += 2)
|
|
{
|
|
options[args[i]] = args[i + 1];
|
|
}
|
|
|
|
if (!options.TryGetValue("--istat", out var istatPath) || !options.TryGetValue("--out", out var outDir))
|
|
{
|
|
Console.WriteLine(Usage);
|
|
return 1;
|
|
}
|
|
|
|
var sheet = ReadFirstSheet(istatPath);
|
|
var provinces = new SortedDictionary<string, (string Name, string Region)>(StringComparer.Ordinal);
|
|
var municipalities = new List<Place>();
|
|
foreach (var row in sheet.Skip(1).Where(r => r.Count > 20 && r[20].Trim().Length > 0))
|
|
{
|
|
var place = new Place(row[20].Trim().ToUpperInvariant(), row[6].Trim(), row[14].Trim(), row[10].Trim(), row[4].Trim());
|
|
if (place.Code.Length != 4 || place.Province.Length != 2 || place.Istat.Length != 6 || place.Name.Contains(';', StringComparison.Ordinal))
|
|
{
|
|
throw new InvalidDataException($"Unexpected row: {string.Join('|', row)}");
|
|
}
|
|
|
|
var province = (row[11].Trim(), place.Region);
|
|
if (provinces.TryGetValue(place.Province, out var known) && known != province)
|
|
{
|
|
throw new InvalidDataException($"Province {place.Province} has two names: {known} and {province}.");
|
|
}
|
|
|
|
provinces[place.Province] = province;
|
|
municipalities.Add(place);
|
|
}
|
|
|
|
if (municipalities.Select(m => m.Code).Distinct(StringComparer.Ordinal).Count() != municipalities.Count)
|
|
{
|
|
throw new InvalidDataException("Duplicate cadastral codes.");
|
|
}
|
|
|
|
var postal = options.TryGetValue("--postal", out var postalPath) ? MatchPostalCodes(postalPath, municipalities) : [];
|
|
|
|
Directory.CreateDirectory(outDir);
|
|
var provinceText = new StringBuilder("sigla;denominazione;regione\n");
|
|
foreach (var (code, (name, region)) in provinces)
|
|
{
|
|
provinceText.Append(CultureInfo.InvariantCulture, $"{code};{name};{region}\n");
|
|
}
|
|
|
|
var municipalityText = new StringBuilder("codice_catastale;denominazione;sigla;regione;codice_istat;cap\n");
|
|
foreach (var m in municipalities.OrderBy(m => m.Istat, StringComparer.Ordinal))
|
|
{
|
|
var codes = postal.TryGetValue(m.Code, out var set) ? Ranges(set) : string.Empty;
|
|
municipalityText.Append(CultureInfo.InvariantCulture, $"{m.Code};{m.Name};{m.Province};{m.Region};{m.Istat};{codes}\n");
|
|
}
|
|
|
|
await File.WriteAllTextAsync(Path.Combine(outDir, "province.csv"), provinceText.ToString(), new UTF8Encoding(false));
|
|
await File.WriteAllTextAsync(Path.Combine(outDir, "comuni.csv"), municipalityText.ToString(), new UTF8Encoding(false));
|
|
Console.WriteLine($"{provinces.Count} provinces, {municipalities.Count} municipalities, {postal.Count} with postal codes.");
|
|
return 0;
|
|
|
|
// The rows of the first sheet of an xlsx, read with the framework only (shared strings and inline strings).
|
|
static List<List<string>> ReadFirstSheet(string path)
|
|
{
|
|
XNamespace ns = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
|
|
using var zip = ZipFile.OpenRead(path);
|
|
var shared = zip.GetEntry("xl/sharedStrings.xml") is { } entry
|
|
? [.. Load(entry).Root!.Elements(ns + "si").Select(si => string.Concat(si.Descendants(ns + "t").Select(t => t.Value)))]
|
|
: new List<string>();
|
|
var sheetEntry = zip.Entries.Where(e => e.FullName.StartsWith("xl/worksheets/sheet", StringComparison.Ordinal)).OrderBy(e => e.FullName, StringComparer.Ordinal).First();
|
|
var rows = new List<List<string>>();
|
|
foreach (var row in Load(sheetEntry).Descendants(ns + "row"))
|
|
{
|
|
var cells = new SortedDictionary<int, string>();
|
|
foreach (var cell in row.Elements(ns + "c"))
|
|
{
|
|
var value = cell.Element(ns + "v")?.Value;
|
|
cells[Column((string)cell.Attribute("r")!)] = (string?)cell.Attribute("t") switch
|
|
{
|
|
"s" when value is not null => shared[int.Parse(value, CultureInfo.InvariantCulture)],
|
|
"inlineStr" => string.Concat(cell.Descendants(ns + "t").Select(t => t.Value)),
|
|
_ => value ?? string.Empty,
|
|
};
|
|
}
|
|
|
|
var width = cells.Count == 0 ? 0 : cells.Keys.Max() + 1;
|
|
rows.Add([.. Enumerable.Range(0, width).Select(i => cells.GetValueOrDefault(i, string.Empty))]);
|
|
}
|
|
|
|
return rows;
|
|
|
|
static XDocument Load(ZipArchiveEntry entry)
|
|
{
|
|
using var stream = entry.Open();
|
|
return XDocument.Load(stream);
|
|
}
|
|
|
|
static int Column(string reference)
|
|
{
|
|
var n = 0;
|
|
foreach (var ch in reference.TakeWhile(char.IsAsciiLetterUpper))
|
|
{
|
|
n = (n * 26) + (ch - 'A' + 1);
|
|
}
|
|
|
|
return n - 1;
|
|
}
|
|
}
|
|
|
|
// GeoNames rows: country, postal code, place, region, region code, province, abbreviation, ...
|
|
static Dictionary<string, SortedSet<string>> MatchPostalCodes(string path, List<Place> municipalities)
|
|
{
|
|
using var zip = ZipFile.OpenRead(path);
|
|
using var reader = new StreamReader(zip.GetEntry("IT.txt")!.Open(), Encoding.UTF8);
|
|
var byNameAndProvince = municipalities.ToLookup(m => (Normalize(m.Name), m.Province));
|
|
var byName = municipalities.ToLookup(m => Normalize(m.Name));
|
|
var result = new Dictionary<string, SortedSet<string>>(StringComparer.Ordinal);
|
|
while (reader.ReadLine() is { } line)
|
|
{
|
|
var fields = line.Split('\t');
|
|
var name = Normalize(fields[2]);
|
|
var match = byNameAndProvince[(name, fields[6])].ToList();
|
|
if (match.Count != 1 && fields[3] == "Sardegna")
|
|
{
|
|
match = [.. byName[name].Where(m => m.Region == "Sardegna")];
|
|
}
|
|
|
|
if (match.Count == 1)
|
|
{
|
|
if (!result.TryGetValue(match[0].Code, out var set))
|
|
{
|
|
result[match[0].Code] = set = new SortedSet<string>(StringComparer.Ordinal);
|
|
}
|
|
|
|
set.Add(fields[1]);
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
// Lower case without accents, apostrophes or hyphens; the Italian part of bilingual names.
|
|
static string Normalize(string name)
|
|
{
|
|
var text = name.Split('/')[0].Normalize(NormalizationForm.FormD);
|
|
var builder = new StringBuilder();
|
|
foreach (var ch in text)
|
|
{
|
|
if (CharUnicodeInfo.GetUnicodeCategory(ch) == UnicodeCategory.NonSpacingMark)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
builder.Append(ch is '-' or '\'' or '`' or '.' ? ' ' : char.ToLowerInvariant(ch));
|
|
}
|
|
|
|
return string.Join(' ', builder.ToString().Split(' ', StringSplitOptions.RemoveEmptyEntries));
|
|
}
|
|
|
|
// 10121,10122,10123 becomes 10121-10123.
|
|
static string Ranges(SortedSet<string> codes)
|
|
{
|
|
var parts = new List<string>();
|
|
string? start = null;
|
|
string? previous = null;
|
|
foreach (var code in codes)
|
|
{
|
|
if (start is not null && int.Parse(code, CultureInfo.InvariantCulture) == int.Parse(previous!, CultureInfo.InvariantCulture) + 1)
|
|
{
|
|
previous = code;
|
|
continue;
|
|
}
|
|
|
|
if (start is not null)
|
|
{
|
|
parts.Add(start == previous ? start : $"{start}-{previous}");
|
|
}
|
|
|
|
start = previous = code;
|
|
}
|
|
|
|
if (start is not null)
|
|
{
|
|
parts.Add(start == previous ? start : $"{start}-{previous}");
|
|
}
|
|
|
|
return string.Join(',', parts);
|
|
}
|
|
|
|
internal sealed record Place(string Code, string Name, string Province, string Region, string Istat);
|