mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 03:38:23 +00:00
Prageeth Panicker
dc3d30acaa
Problem: Every compose service logs to journald through sys-svc-container/templates/base.yml.j2. The daily journal health check grepped the whole journal for "error", so routine application errors from containers triggered the host alert (email/Telegram) and buried real host problems. Change: - Read the last day of the journal as JSON (MESSAGE, CONTAINER_ID, SYSLOG_IDENTIFIER, _SYSTEMD_UNIT) and skip every entry that carries a CONTAINER_ID field. Docker daemon and system errors are still reported. - Report each match as one readable "source: message" line instead of raw journal output. - Stream the journalctl output line by line instead of buffering a whole day of logs in memory. - Exit 2 with journalctl's stderr if journalctl itself fails, instead of silently reporting "no errors". - Rewrite the check in Python (files/python/script.py, deployed by sys-service like the other sys-ctl-hlth roles) and decode non-UTF-8 messages. - Fix the success message, which claimed "All docker containers are healthy". Impact: Container log errors no longer raise this host alert on any installation. Container health stays covered by sys-ctl-hlth-container; application log analysis belongs in a log platform such as Wazuh. system_service_suppress_flush is unchanged until a deploy confirms that no host errors remain. Tests: Unit tests run the script against a fake journalctl (8 cases): container error ignored, daemon error reported, identifier fallback, non-UTF-8 message, journalctl failure, many container lines streamed past, clean journal passes. The core cases fail against the previous script.
85 lines
2.3 KiB
Python
Executable File
85 lines
2.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
JOURNALCTL = [
|
|
"journalctl",
|
|
"--since",
|
|
"1 day ago",
|
|
"--no-pager",
|
|
"--output=json",
|
|
"--output-fields=MESSAGE,CONTAINER_ID,SYSLOG_IDENTIFIER,_SYSTEMD_UNIT",
|
|
]
|
|
|
|
|
|
def message_text(value):
|
|
"""Return a journal MESSAGE as one line of text.
|
|
|
|
Args:
|
|
value: the MESSAGE field; journald encodes non-UTF-8 messages as a list of bytes.
|
|
|
|
Returns:
|
|
The message with line breaks collapsed to spaces.
|
|
"""
|
|
if isinstance(value, list):
|
|
value = bytes(value).decode("utf-8", errors="replace")
|
|
return " ".join(str(value).splitlines())
|
|
|
|
|
|
def host_errors(lines):
|
|
"""Yield `source: message` for every non-container entry that mentions an error.
|
|
|
|
Args:
|
|
lines: iterable of journalctl JSON lines, one entry per line, consumed lazily.
|
|
|
|
Returns:
|
|
Iterator of readable error lines.
|
|
"""
|
|
for line in lines:
|
|
if not line.strip():
|
|
continue
|
|
entry = json.loads(line)
|
|
if "CONTAINER_ID" in entry:
|
|
continue
|
|
message = message_text(entry.get("MESSAGE", ""))
|
|
if "error" not in message.lower():
|
|
continue
|
|
source = (
|
|
entry.get("_SYSTEMD_UNIT") or entry.get("SYSLOG_IDENTIFIER") or "unknown"
|
|
)
|
|
yield f"{source}: {message}"
|
|
|
|
|
|
def main():
|
|
print("Checking journalctl for error messages outside container logs...")
|
|
# stderr goes to a file, not a pipe: an unread stderr pipe that fills up would block journalctl while stdout is still being read.
|
|
with tempfile.TemporaryFile(
|
|
mode="w+", encoding="utf-8", errors="replace"
|
|
) as stderr:
|
|
with subprocess.Popen(
|
|
JOURNALCTL,
|
|
stdout=subprocess.PIPE,
|
|
stderr=stderr,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
) as journal:
|
|
errors = list(host_errors(journal.stdout))
|
|
if journal.returncode != 0:
|
|
stderr.seek(0)
|
|
print(
|
|
f"journalctl failed with exit code {journal.returncode}: {stderr.read().strip()}"
|
|
)
|
|
return 2
|
|
if errors:
|
|
print("Some errors were found:")
|
|
print("\n".join(errors))
|
|
return 1
|
|
print("No errors found in the journal.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|