Files
computer-playbook/tests/unit/python/roles/sys-ctl-hlth-journalctl/files/test_script.py
T
Prageeth Panicker dc3d30acaa fix(sys-ctl-hlth-journalctl): skip container logs in the host journal check
Problem:
Every compose service logs to journald through
sys-svc-container/templates/base.yml.j2. The daily journal health
check grepped the whole journal for "error", so routine application
errors from containers triggered the host alert (email/Telegram) and
buried real host problems.

Change:
- Read the last day of the journal as JSON (MESSAGE, CONTAINER_ID,
  SYSLOG_IDENTIFIER, _SYSTEMD_UNIT) and skip every entry that carries
  a CONTAINER_ID field. Docker daemon and system errors are still
  reported.
- Report each match as one readable "source: message" line instead
  of raw journal output.
- Stream the journalctl output line by line instead of buffering a
  whole day of logs in memory.
- Exit 2 with journalctl's stderr if journalctl itself fails, instead
  of silently reporting "no errors".
- Rewrite the check in Python (files/python/script.py, deployed by
  sys-service like the other sys-ctl-hlth roles) and decode non-UTF-8
  messages.
- Fix the success message, which claimed "All docker containers are
  healthy".

Impact:
Container log errors no longer raise this host alert on any
installation. Container health stays covered by sys-ctl-hlth-container;
application log analysis belongs in a log platform such as Wazuh.
system_service_suppress_flush is unchanged until a deploy confirms
that no host errors remain.

Tests:
Unit tests run the script against a fake journalctl (8 cases):
container error ignored, daemon error reported, identifier fallback,
non-UTF-8 message, journalctl failure, many container lines streamed
past, clean journal passes. The core cases fail against the previous
script.
2026-10-16 14:41:05 +00:00

134 lines
4.5 KiB
Python

import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from utils import PROJECT_ROOT
SCRIPT = PROJECT_ROOT / "roles/sys-ctl-hlth-journalctl/files/python/script.py"
JOURNALCTL_STUB = """#!/usr/bin/env bash
printf '%s\\n' "$*" >"${STUB_DIR}/args"
cat "${STUB_DIR}/journal"
if [ "${STUB_RC}" -ne 0 ]; then
\techo "journalctl: unrecognized option '--output-fields'" >&2
fi
exit "${STUB_RC}"
"""
APP_ERROR = {
"MESSAGE": "ERROR: pixelfed queue failed",
"CONTAINER_ID": "0123456789ab",
"SYSLOG_IDENTIFIER": "pixelfed",
"_SYSTEMD_UNIT": "docker.service",
}
DAEMON_ERROR = {
"MESSAGE": 'level=error msg="failed to start container"',
"SYSLOG_IDENTIFIER": "dockerd",
"_SYSTEMD_UNIT": "docker.service",
}
KERNEL_ERROR = {"MESSAGE": "EXT4-fs error (device sda1)", "SYSLOG_IDENTIFIER": "kernel"}
BINARY_ERROR = {
"MESSAGE": list(b"I/O error \xff on disk"),
"_SYSTEMD_UNIT": "smartd.service",
}
SYSTEM_INFO = {
"MESSAGE": "Started Daily apt download activities.",
"_SYSTEMD_UNIT": "init.scope",
}
class TestHlthJournalctlScript(unittest.TestCase):
def run_script(self, *entries, rc=0):
"""Run the health script against a stubbed journalctl.
Args:
entries: journal entries the stub prints, one JSON object per line.
rc: exit code of the stubbed journalctl.
Returns:
Tuple of the exit code, the script output, and the journalctl arguments.
"""
with tempfile.TemporaryDirectory() as tmp:
bin_dir = Path(tmp)
stub = bin_dir / "journalctl"
stub.write_text(JOURNALCTL_STUB)
stub.chmod(0o755)
(bin_dir / "journal").write_text(
"".join(json.dumps(entry) + "\n" for entry in entries)
)
env = dict(os.environ)
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
env["STUB_DIR"] = tmp
env["STUB_RC"] = str(rc)
done = subprocess.run(
[sys.executable, str(SCRIPT)],
env=env,
capture_output=True,
text=True,
check=False,
)
args = (bin_dir / "args").read_text()
return done.returncode, done.stdout, args
def test_reads_the_journal_as_json_with_the_source_fields(self):
_, _, args = self.run_script(SYSTEM_INFO)
self.assertIn("--output=json", args)
self.assertIn(
"--output-fields=MESSAGE,CONTAINER_ID,SYSLOG_IDENTIFIER,_SYSTEMD_UNIT",
args,
)
def test_an_application_error_from_a_container_is_ignored(self):
code, _, _ = self.run_script(APP_ERROR, SYSTEM_INFO)
self.assertEqual(code, 0)
def test_a_docker_daemon_error_is_reported_with_its_unit(self):
code, output, _ = self.run_script(APP_ERROR, DAEMON_ERROR)
self.assertEqual(code, 1)
self.assertIn(
'docker.service: level=error msg="failed to start container"', output
)
self.assertNotIn("pixelfed queue failed", output)
def test_an_entry_without_unit_is_named_by_its_identifier(self):
code, output, _ = self.run_script(KERNEL_ERROR)
self.assertEqual(code, 1)
self.assertIn("kernel: EXT4-fs error (device sda1)", output)
def test_a_non_utf8_message_is_decoded_and_reported(self):
code, output, _ = self.run_script(BINARY_ERROR)
self.assertEqual(code, 1)
self.assertIn("smartd.service: I/O error", output)
def test_many_container_errors_are_streamed_past_and_not_reported(self):
container_lines = [
{**APP_ERROR, "MESSAGE": f"ERROR: pixelfed request {n} failed"}
for n in range(20000)
]
code, output, _ = self.run_script(*container_lines, DAEMON_ERROR)
self.assertEqual(code, 1)
self.assertNotIn("pixelfed request", output)
self.assertEqual(
output.splitlines()[-1],
'docker.service: level=error msg="failed to start container"',
)
self.assertEqual(len(output.splitlines()), 3)
def test_a_journal_without_errors_passes(self):
code, _, _ = self.run_script(SYSTEM_INFO)
self.assertEqual(code, 0)
def test_a_failing_journalctl_fails_the_check(self):
code, output, _ = self.run_script(rc=1)
self.assertEqual(code, 2)
self.assertIn("journalctl failed with exit code 1", output)
self.assertIn("unrecognized option '--output-fields'", output)
if __name__ == "__main__":
unittest.main()