Merge pull request #974 from pragepani/fix/sys-ctl-hlth-journalctl/skip-container-logs

fix(sys-ctl-hlth-journalctl): skip container logs in the host journal…

roles/sys-ctl-hlth-journalctl/README.md

@@ -2,11 +2,13 @@
 
 ## Description
 
-Scans `journalctl` over the last day for “error” entries and alerts if any are found.
+Scans `journalctl` over the last day for “error” entries outside container logs and alerts if any are found.
 
 ## Overview
 
 This role searches the systemd journal for errors over the past day and alerts if any are found.
+Container output is skipped, because every compose service logs to journald and application errors would otherwise trigger the host check.
+Docker daemon and system errors are still reported.
 
 ## Cosmos
 
@@ -23,8 +25,11 @@ Solid `1:1` edges are fixed relationships; dashed `0..1` edges are conditional (
 
 ## Features
 
-- Runs `journalctl --since '1 day ago' | grep -i error`.
-- Exits non-zero on matches.
+- Reads the last day of the journal as JSON with every field in full (`--all`) and drops every entry that carries a `CONTAINER_ID` field.
+- Joins a repeated `MESSAGE` field into one line before matching.
+- Matches the remaining messages case-insensitively against `error`.
+- Prints one `source: message` line per match, naming the systemd unit or, without one, the syslog identifier, so alerts show who logged the error.
+- Exits non-zero on matches, and also when `journalctl` itself fails, so a broken check never passes silently.
 - Scheduled via systemd timer.
 - Alerts via `sys-ctl-alm-compose` on detection.
 

roles/sys-ctl-hlth-journalctl/files/python/script.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+import json
+import subprocess
+import sys
+import tempfile
+
+JOURNALCTL = [
+    "journalctl",
+    "--since",
+    "1 day ago",
+    "--no-pager",
+    "--all",
+    "--output=json",
+    "--output-fields=MESSAGE,CONTAINER_ID,SYSLOG_IDENTIFIER,_SYSTEMD_UNIT",
+]
+
+
+def message_text(value):
+    """Return a journal MESSAGE as one line of text.
+
+    Args:
+        value: the MESSAGE field; journald encodes non-UTF-8 messages as a list of bytes and a repeated field as a list of its values.
+
+    Returns:
+        The message with line breaks collapsed to spaces.
+    """
+    if isinstance(value, list) and all(isinstance(item, int) for item in value):
+        value = bytes(value).decode("utf-8", errors="replace")
+    elif isinstance(value, list):
+        value = "\n".join(message_text(item) for item in value)
+    return " ".join(str(value).splitlines())
+
+
+def host_errors(lines):
+    """Yield `source: message` for every non-container entry that mentions an error.
+
+    Args:
+        lines: iterable of journalctl JSON lines, one entry per line, consumed lazily.
+
+    Returns:
+        Iterator of readable error lines.
+    """
+    for line in lines:
+        if not line.strip():
+            continue
+        entry = json.loads(line)
+        if "CONTAINER_ID" in entry:
+            continue
+        message = message_text(entry.get("MESSAGE", ""))
+        if "error" not in message.lower():
+            continue
+        source = (
+            entry.get("_SYSTEMD_UNIT") or entry.get("SYSLOG_IDENTIFIER") or "unknown"
+        )
+        yield f"{source}: {message}"
+
+
+def main():
+    print("Checking journalctl for error messages outside container logs...")
+    with tempfile.TemporaryFile(
+        mode="w+", encoding="utf-8", errors="replace"
+    ) as stderr:
+        with subprocess.Popen(
+            JOURNALCTL,
+            stdout=subprocess.PIPE,
+            stderr=stderr,
+            encoding="utf-8",
+            errors="replace",
+        ) as journal:
+            errors = list(host_errors(journal.stdout))
+        if journal.returncode != 0:
+            stderr.seek(0)
+            print(
+                f"journalctl failed with exit code {journal.returncode}: {stderr.read().strip()}"
+            )
+            return 2
+    if errors:
+        print("Some errors were found:")
+        print("\n".join(errors))
+        return 1
+    print("No errors found in the journal.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

roles/sys-ctl-hlth-journalctl/files/shell/script.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-echo "Checking journalctl for error messages..."
-journalctl_errors="$(journalctl --since '1 day ago' --no-pager | grep -i 'error')"
-if [ -n "$journalctl_errors" ]
-then 
-   echo "Some errors where found: $journalctl_errors"
-   exit 1
-fi
-echo "All docker containers are healthy."
-exit 0

tests/unit/python/roles/sys-ctl-hlth-journalctl/files/test_script.py

@@ -0,0 +1,147 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+from utils import PROJECT_ROOT
+
+SCRIPT = PROJECT_ROOT / "roles/sys-ctl-hlth-journalctl/files/python/script.py"
+
+JOURNALCTL_STUB = """#!/usr/bin/env bash
+printf '%s\\n' "$*" >"${STUB_DIR}/args"
+cat "${STUB_DIR}/journal"
+if [ "${STUB_RC}" -ne 0 ]; then
+\techo "journalctl: unrecognized option '--output-fields'" >&2
+fi
+exit "${STUB_RC}"
+"""
+
+APP_ERROR = {
+    "MESSAGE": "ERROR: pixelfed queue failed",
+    "CONTAINER_ID": "0123456789ab",
+    "SYSLOG_IDENTIFIER": "pixelfed",
+    "_SYSTEMD_UNIT": "docker.service",
+}
+DAEMON_ERROR = {
+    "MESSAGE": 'level=error msg="failed to start container"',
+    "SYSLOG_IDENTIFIER": "dockerd",
+    "_SYSTEMD_UNIT": "docker.service",
+}
+KERNEL_ERROR = {"MESSAGE": "EXT4-fs error (device sda1)", "SYSLOG_IDENTIFIER": "kernel"}
+BINARY_ERROR = {
+    "MESSAGE": list(b"I/O error \xff on disk"),
+    "_SYSTEMD_UNIT": "smartd.service",
+}
+REPEATED_ERROR = {
+    "MESSAGE": ["unit started", list(b"disk error \xff on sdb")],
+    "_SYSTEMD_UNIT": "zfs-import.service",
+}
+SYSTEM_INFO = {
+    "MESSAGE": "Started Daily apt download activities.",
+    "_SYSTEMD_UNIT": "init.scope",
+}
+
+
+class TestHlthJournalctlScript(unittest.TestCase):
+    def run_script(self, *entries, rc=0):
+        """Run the health script against a stubbed journalctl.
+
+        Args:
+            entries: journal entries the stub prints, one JSON object per line.
+            rc: exit code of the stubbed journalctl.
+
+        Returns:
+            Tuple of the exit code, the script output, and the journalctl arguments.
+        """
+        with tempfile.TemporaryDirectory() as tmp:
+            bin_dir = Path(tmp)
+            stub = bin_dir / "journalctl"
+            stub.write_text(JOURNALCTL_STUB)
+            stub.chmod(0o755)
+            (bin_dir / "journal").write_text(
+                "".join(json.dumps(entry) + "\n" for entry in entries)
+            )
+            env = dict(os.environ)
+            env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
+            env["STUB_DIR"] = tmp
+            env["STUB_RC"] = str(rc)
+            done = subprocess.run(
+                [sys.executable, str(SCRIPT)],
+                env=env,
+                capture_output=True,
+                text=True,
+                check=False,
+            )
+            args = (bin_dir / "args").read_text()  # nocheck: cache-read tempdir fixture
+            return done.returncode, done.stdout, args
+
+    def test_reads_the_journal_as_json_with_the_source_fields(self):
+        _, _, args = self.run_script(SYSTEM_INFO)
+        self.assertIn("--output=json", args)
+        self.assertIn(
+            "--output-fields=MESSAGE,CONTAINER_ID,SYSLOG_IDENTIFIER,_SYSTEMD_UNIT",
+            args,
+        )
+
+    def test_reads_every_field_in_full(self):
+        _, _, args = self.run_script(SYSTEM_INFO)
+        self.assertIn("--all", args.split())
+
+    def test_a_repeated_message_field_is_joined_and_reported(self):
+        code, output, _ = self.run_script(REPEATED_ERROR, DAEMON_ERROR)
+        self.assertEqual(code, 1)
+        self.assertIn("zfs-import.service: unit started disk error", output)
+        self.assertIn('docker.service: level=error msg="failed to start', output)
+
+    def test_an_application_error_from_a_container_is_ignored(self):
+        code, _, _ = self.run_script(APP_ERROR, SYSTEM_INFO)
+        self.assertEqual(code, 0)
+
+    def test_a_docker_daemon_error_is_reported_with_its_unit(self):
+        code, output, _ = self.run_script(APP_ERROR, DAEMON_ERROR)
+        self.assertEqual(code, 1)
+        self.assertIn(
+            'docker.service: level=error msg="failed to start container"', output
+        )
+        self.assertNotIn("pixelfed queue failed", output)
+
+    def test_an_entry_without_unit_is_named_by_its_identifier(self):
+        code, output, _ = self.run_script(KERNEL_ERROR)
+        self.assertEqual(code, 1)
+        self.assertIn("kernel: EXT4-fs error (device sda1)", output)
+
+    def test_a_non_utf8_message_is_decoded_and_reported(self):
+        code, output, _ = self.run_script(BINARY_ERROR)
+        self.assertEqual(code, 1)
+        self.assertIn("smartd.service: I/O error", output)
+
+    def test_many_container_errors_are_streamed_past_and_not_reported(self):
+        container_lines = [
+            {**APP_ERROR, "MESSAGE": f"ERROR: pixelfed request {n} failed"}
+            for n in range(20000)
+        ]
+        code, output, _ = self.run_script(*container_lines, DAEMON_ERROR)
+        self.assertEqual(code, 1)
+        self.assertNotIn("pixelfed request", output)
+        self.assertEqual(
+            output.splitlines()[-1],
+            'docker.service: level=error msg="failed to start container"',
+        )
+        self.assertEqual(len(output.splitlines()), 3)
+
+    def test_a_journal_without_errors_passes(self):
+        code, _, _ = self.run_script(SYSTEM_INFO)
+        self.assertEqual(code, 0)
+
+    def test_a_failing_journalctl_fails_the_check(self):
+        code, output, _ = self.run_script(rc=1)
+        self.assertEqual(code, 2)
+        self.assertIn("journalctl failed with exit code 1", output)
+        self.assertIn("unrecognized option '--output-fields'", output)
+
+
+if __name__ == "__main__":
+    unittest.main()