The server log of a failing job shows every restart as 'untracked child process exited with exit code 2'. A health check that outlives its 3-second timeout leaves pg_isready to PostgreSQL, which is PID 1 in its container; when pg_isready exits the postmaster takes it for a crashed backend and restarts every connection. --init reaps such orphans, and pg_isready -t 5 under a 15-second timeout is never cut off. Same change in docker-compose.yml. ADR 0015. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
146 lines
6.9 KiB
C#
146 lines
6.9 KiB
C#
using System.Globalization;
|
|
using System.Net.Http.Headers;
|
|
using System.Net.Sockets;
|
|
using System.Text;
|
|
using System.Text.Json.Nodes;
|
|
|
|
/// <summary>
|
|
/// Prints what the CI cannot see from inside the job: the log, state and memory of the service containers next to this
|
|
/// one (the PostgreSQL of the tests). It talks to the Docker Engine API through the socket the runner mounts in the job
|
|
/// container, so it needs neither the docker client nor anything else to install (ADR 0004).
|
|
/// </summary>
|
|
internal static class ServiceLog
|
|
{
|
|
private const string Socket = "/var/run/docker.sock";
|
|
|
|
public static int Print(string imagePrefix)
|
|
{
|
|
if (!File.Exists(Socket))
|
|
{
|
|
Console.WriteLine($"Nessun {Socket}: il registro dei servizi si legge solo nella CI.");
|
|
return 0;
|
|
}
|
|
|
|
using var http = new HttpClient(new SocketsHttpHandler
|
|
{
|
|
ConnectCallback = async (_, token) =>
|
|
{
|
|
var socket = new System.Net.Sockets.Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified);
|
|
await socket.ConnectAsync(new UnixDomainSocketEndPoint(Socket), token).ConfigureAwait(false);
|
|
return new NetworkStream(socket, ownsSocket: true);
|
|
},
|
|
})
|
|
{
|
|
BaseAddress = new Uri("http://docker/"),
|
|
Timeout = TimeSpan.FromMinutes(2),
|
|
};
|
|
|
|
var info = Json(http, "info");
|
|
Console.WriteLine($"Host: {info["Name"]}, {info["NCPU"]} CPU, {Mib(info["MemTotal"])} di memoria, kernel {info["KernelVersion"]}");
|
|
|
|
var self = Json(http, $"containers/{Uri.EscapeDataString(Environment.MachineName)}/json");
|
|
var networks = self["NetworkSettings"]?["Networks"]?.AsObject().Select(pair => pair.Key).ToList() ?? [];
|
|
var found = 0;
|
|
foreach (var network in networks)
|
|
{
|
|
var filter = Uri.EscapeDataString($$"""{"network":["{{network}}"]}""");
|
|
foreach (var container in Json(http, $"containers/json?all=1&filters={filter}").AsArray())
|
|
{
|
|
var image = container?["Image"]?.GetValue<string>() ?? "";
|
|
if (!image.StartsWith(imagePrefix, StringComparison.Ordinal))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
found++;
|
|
var id = container!["Id"]!.GetValue<string>();
|
|
Describe(http, id, image);
|
|
}
|
|
}
|
|
|
|
if (found == 0)
|
|
{
|
|
Console.WriteLine($"Nessun contenitore '{imagePrefix}*' sulle reti {string.Join(", ", networks)}.");
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
private static void Describe(HttpClient http, string id, string image)
|
|
{
|
|
var inspect = Json(http, $"containers/{id}/json");
|
|
var state = inspect["State"];
|
|
var host = inspect["HostConfig"];
|
|
Console.WriteLine();
|
|
Console.WriteLine($"== {image} {id[..12]}");
|
|
Console.WriteLine($"Stato: {state?["Status"]}, avviato {state?["StartedAt"]}, riavvii {inspect["RestartCount"]}, OOMKilled {state?["OOMKilled"]}, uscita {state?["ExitCode"]}");
|
|
Console.WriteLine($"Limiti: memoria {Mib(host?["Memory"])}, /dev/shm {Mib(host?["ShmSize"])}, init {host?["Init"]?.ToString() ?? "no"}");
|
|
Console.WriteLine($"Controllo di salute: {inspect["Config"]?["Healthcheck"]?["Test"]?.ToJsonString()}, timeout {(inspect["Config"]?["Healthcheck"]?["Timeout"]?.GetValue<long>() ?? 0) / 1_000_000_000} s");
|
|
|
|
if (state?["Running"]?.GetValue<bool>() == true)
|
|
{
|
|
var stats = Json(http, $"containers/{id}/stats?stream=false");
|
|
var memory = stats["memory_stats"];
|
|
Console.WriteLine($"Memoria ora: {Mib(memory?["usage"])}, limite visto dal contenitore {Mib(memory?["limit"])}, massimo {Mib(memory?["max_usage"])}");
|
|
}
|
|
|
|
// Out-of-memory kills and signals as Docker saw them, from the container's start until now.
|
|
var since = DateTimeOffset.Parse(state?["StartedAt"]?.GetValue<string>() ?? "", CultureInfo.InvariantCulture).ToUnixTimeSeconds();
|
|
var until = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
|
var events = Uri.EscapeDataString($$"""{"container":["{{id}}"]}""");
|
|
using (var answer = http.GetAsync(new Uri($"events?since={since}&until={until}&filters={events}", UriKind.Relative)).GetAwaiter().GetResult())
|
|
{
|
|
answer.EnsureSuccessStatusCode();
|
|
var lines = answer.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
|
.Split('\n', StringSplitOptions.RemoveEmptyEntries)
|
|
.Select(line => JsonNode.Parse(line))
|
|
.Where(e => e?["Action"]?.GetValue<string>() is not { } action
|
|
|| !(action.StartsWith("exec_", StringComparison.Ordinal) || action == "health_status: healthy"))
|
|
.Select(e => $" {DateTimeOffset.FromUnixTimeSeconds(e?["time"]?.GetValue<long>() ?? 0):O} {e?["Action"]}")
|
|
.ToList();
|
|
Console.WriteLine($"Eventi di Docker: {(lines.Count == 0 ? "nessuno" : "")}");
|
|
lines.ForEach(Console.WriteLine);
|
|
}
|
|
|
|
Console.WriteLine("-- registro");
|
|
using var response = http.GetAsync(new Uri($"containers/{id}/logs?stdout=1&stderr=1×tamps=1", UriKind.Relative)).GetAwaiter().GetResult();
|
|
response.EnsureSuccessStatusCode();
|
|
var bytes = response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult();
|
|
var raw = response.Content.Headers.ContentType?.MediaType == "application/vnd.docker.raw-stream";
|
|
Console.Write(raw ? Encoding.UTF8.GetString(bytes) : Demultiplex(bytes));
|
|
Console.WriteLine("-- fine del registro");
|
|
}
|
|
|
|
// Without a terminal the log arrives in frames: one byte for the stream, three zeros, four bytes of length.
|
|
private static string Demultiplex(byte[] bytes)
|
|
{
|
|
var text = new StringBuilder();
|
|
var at = 0;
|
|
while (at + 8 <= bytes.Length)
|
|
{
|
|
var length = (bytes[at + 4] << 24) | (bytes[at + 5] << 16) | (bytes[at + 6] << 8) | bytes[at + 7];
|
|
var size = Math.Min(length, bytes.Length - at - 8);
|
|
text.Append(Encoding.UTF8.GetString(bytes, at + 8, size));
|
|
at += 8 + length;
|
|
}
|
|
|
|
return text.ToString();
|
|
}
|
|
|
|
private static JsonNode Json(HttpClient http, string path)
|
|
{
|
|
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(path, UriKind.Relative));
|
|
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
|
using var response = http.Send(request);
|
|
response.EnsureSuccessStatusCode();
|
|
using var stream = response.Content.ReadAsStream();
|
|
return JsonNode.Parse(stream) ?? new JsonObject();
|
|
}
|
|
|
|
private static string Mib(JsonNode? bytes)
|
|
{
|
|
var value = bytes?.GetValue<long>() ?? 0;
|
|
return value == 0 ? "nessuno" : (value / 1048576.0).ToString("0", CultureInfo.InvariantCulture) + " MiB";
|
|
}
|
|
}
|