mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 10:59:59 +00:00
feat(diagnostics): capture the logs an application writes to a file
The collector took `logs` from every container, which is stdout and stderr and nothing else. An application that logs to a file left the bundle with a status code and no trace of why. That is what stopped the snipe-it investigation in run 33317483326. Its MCP adapter reported `UNREACHABLE read call snipeit_get_license_summary: the provider answered 500`, and Laravel had written the exception to storage/logs inside the container, where `logs` never looks. Three separate root causes were proposed from static reading and all three were refuted -- the relation exists, inherited from SnipeModel; the group permissions are stored exactly as decodePermissions expects; the controller is identical at the deployed tag -- and the job is still unexplained. The collector already execs into every container for resolv.conf, systemctl and journalctl, so the gap was the application's own files, not the ability to reach them. find drives the capture through -exec rather than a loop over $(find ...): the first version split on whitespace and broke on paths containing spaces, which a probe against a directory with two of them showed immediately. The per-container and per-service captures fold into tables because the file sits at the 500-line limit. Splitting it is not an option: this script copies itself into every container it inspects and runs there standalone, so an import of a sibling module resolves to nothing -- an attempt at that broke seventeen tests. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a372d5aa0f
commit
8434d35d82
1 file changed
+22
-25
@@ -40,6 +40,7 @@ _NESTED_TIMEOUT = 600
|
||||
_TAR_TIMEOUT = 300
|
||||
_SELF_IN_CONTAINER = "/tmp/rescue-self.py" # noqa: S108 - fixed staging path inside the inspected container
|
||||
_LOCAL_DUMPS_ENV = "INFINITO_RESCUE_LOCAL_DUMPS_DIR"
|
||||
_APP_LOG_TAIL = "find /var/log /var/www /opt /srv -xdev -type f -name '*.log' -size -20M -mmin -360 -exec sh -c 'echo \"===== $1\"; tail -n 400 \"$1\"' _ {} ';' 2>/dev/null"
|
||||
_PROBE_HOSTS = ("deb.debian.org", "ghcr.io", "repo.packagist.org")
|
||||
_INZONE_PROBE = "getent hosts rescue-probe.$(awk -F/ '/^address=/{print $2;exit}' /etc/dnsmasq.conf)"
|
||||
|
||||
@@ -330,23 +331,22 @@ def collect_runtime(out: Path, rt: str) -> tuple[list[str], list[str]]:
|
||||
safe = sanitize(name)
|
||||
capture(out / "containers", f"{safe}.log", [rt, "logs", name])
|
||||
capture(out / "containers", f"{safe}.inspect.json", [rt, "inspect", name])
|
||||
capture(
|
||||
out / "containers",
|
||||
f"{safe}.resolv-conf.txt",
|
||||
[rt, "exec", name, "cat", "/etc/resolv.conf"],
|
||||
timeout=_PROBE_TIMEOUT,
|
||||
)
|
||||
capture(
|
||||
out / "containers",
|
||||
f"{safe}.systemctl.txt",
|
||||
[rt, "exec", name, "systemctl", "status", "--all", "--no-pager"],
|
||||
timeout=_PROBE_TIMEOUT,
|
||||
)
|
||||
capture(
|
||||
out / "containers",
|
||||
f"{safe}.journal.txt",
|
||||
[rt, "exec", name, "journalctl", "-b", "--no-pager"],
|
||||
)
|
||||
for suffix, argv, probe_timeout in (
|
||||
("resolv-conf.txt", ["cat", "/etc/resolv.conf"], _PROBE_TIMEOUT),
|
||||
(
|
||||
"systemctl.txt",
|
||||
["systemctl", "status", "--all", "--no-pager"],
|
||||
_PROBE_TIMEOUT,
|
||||
),
|
||||
("journal.txt", ["journalctl", "-b", "--no-pager"], _EXEC_TIMEOUT),
|
||||
("app-logs.txt", ["sh", "-c", _APP_LOG_TAIL], _PROBE_TIMEOUT),
|
||||
):
|
||||
capture(
|
||||
out / "containers",
|
||||
f"{safe}.{suffix}",
|
||||
[rt, "exec", name, *argv],
|
||||
timeout=probe_timeout,
|
||||
)
|
||||
if "postgres" in name:
|
||||
capture(
|
||||
out / "containers",
|
||||
@@ -366,14 +366,11 @@ def collect_runtime(out: Path, rt: str) -> tuple[list[str], list[str]]:
|
||||
services = list_lines([rt, "service", "ls", "--format", "{{.Name}}"])
|
||||
for svc in services:
|
||||
safe = sanitize(svc)
|
||||
capture(
|
||||
out / "services", f"{safe}.ps.txt", [rt, "service", "ps", "--no-trunc", svc]
|
||||
)
|
||||
capture(
|
||||
out / "services",
|
||||
f"{safe}.log",
|
||||
[rt, "service", "logs", "--no-task-ids", svc],
|
||||
)
|
||||
for suffix, argv in (
|
||||
("ps.txt", ["service", "ps", "--no-trunc", svc]),
|
||||
("log", ["service", "logs", "--no-task-ids", svc]),
|
||||
):
|
||||
capture(out / "services", f"{safe}.{suffix}", [rt, *argv])
|
||||
return containers, services
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user