computer-playbook/roles/sys-ctl-rpr-container-soft/files/python/script.py

#!/usr/bin/env python3
"""
Restart Docker-Compose configurations with exited or unhealthy containers.

STRICT mode: Resolve the Compose project exclusively via Docker labels
(com.docker.compose.project and com.docker.compose.project.working_dir).
No container-name fallback. If labels are missing or Docker is unavailable,
the script records an error for that container.

All shell interactions that matter for tests go through print_bash()
so they can be monkeypatched in unit tests.

This variant uses the central Infinito.Nexus compose wrapper:
  compose
which auto-adds:
- -f compose.yml
- -f compose.override.yml (if present)
- -f compose.ca.override.yml (if present)
- --env-file (if .env or .env/env exists)
"""

import argparse
import os
import subprocess
import syslog
import time

# ---------------------------
# Shell helpers
# ---------------------------


def bash(command: str) -> list[str]:
    print(command)
    process = subprocess.Popen(  # noqa: S602
        [command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
    )
    out, err = process.communicate()
    stdout = out.splitlines()
    stderr = err.decode("utf-8", errors="replace").strip()
    output = [line.decode("utf-8", errors="replace") for line in stdout]
    if process.returncode > 0:
        print(command, out, err)
        raise RuntimeError(stderr or f"Command failed with code {process.returncode}")
    return output


def list_to_string(lst: list[str]) -> str:
    return " ".join(lst)


def log_kill(action: str, target: str) -> None:
    syslog.openlog(ident="infinito-kill")
    syslog.syslog(syslog.LOG_WARNING, f"sys-ctl-rpr-container-soft: {action} {target}")


def print_bash(command: str) -> list[str]:
    """
    Wrapper around bash() that echoes combined output for easier debugging
    and can be monkeypatched in tests.
    """
    output = bash(command)
    if output:
        print(list_to_string(output))
    return output


# ---------------------------
# Compose wrapper helpers
# ---------------------------


def compose_cmd(subcmd: str, project_path: str, project_name: str) -> str:
    """
    Build an compose command that auto-adds env + override files.
    Example: compose --chdir "/opt/compose/foo" --project "foo" restart
    """
    return f'compose --chdir "{project_path}" --project "{project_name}" {subcmd}'


# ---------------------------
# Business logic
# ---------------------------


def normalize_services_arg(raw: list[str] | None, raw_str: str | None) -> list[str]:
    """
    Accept either:
      - multiple --manipulation SERVICE flags (nargs='*')
      - a single --manipulation-string "svc1 svc2 ..." (space or comma separated)
    """
    if raw:
        return [s for s in raw if s.strip()]
    if raw_str:
        parts = [p.strip() for chunk in raw_str.split(",") for p in chunk.split()]
        return [p for p in parts if p]
    return []


def wait_while_manipulation_running(
    services: list[str],
    waiting_time: int = 600,
    timeout: int | None = None,
) -> None:
    """
    Wait until none of the given services are active anymore.
    Stops waiting if timeout (in seconds) is reached.
    """
    if not services:
        print("No manipulation services provided. Continuing without wait.")
        return

    start = time.time()
    while True:
        any_active = False
        for svc in services:
            res = subprocess.run(  # noqa: S602
                f"systemctl is-active --quiet {svc}", shell=True, check=False
            )
            if res.returncode == 0:
                any_active = True
                break

        if any_active:
            elapsed = time.time() - start
            if timeout and elapsed >= timeout:
                print(
                    f"Timeout ({timeout}s) reached while waiting for services. Continuing anyway."
                )
                break
            print(
                f"Manipulation service is running. Trying again in {waiting_time} seconds."
            )
            time.sleep(waiting_time)
        else:
            print("No blocking service is running.")
            break


def get_compose_project_info(container: str) -> tuple[str, str]:
    """
    Resolve project name and working dir from Docker labels.
    STRICT: Raises RuntimeError if labels are missing/unreadable.
    On failure: dump *everything* Docker knows about the container.
    """
    out_project = print_bash(
        f"container inspect --type container -f '{{{{ index .Config.Labels \"com.docker.compose.project\" }}}}' {container}"
    )
    out_workdir = print_bash(
        f"container inspect --type container -f '{{{{ index .Config.Labels \"com.docker.compose.project.working_dir\" }}}}' {container}"
    )

    project = out_project[0].strip() if out_project else ""
    workdir = out_workdir[0].strip() if out_workdir else ""

    if not project or not workdir:
        print("DEBUG: compose label lookup failed – dumping the full state")
        try:
            print_bash(f"container inspect --type container {container}")
        except Exception as e:
            print(f"DEBUG: the state dump failed completely: {e}")

    if not project:
        raise RuntimeError(f"No compose project label found for container {container}")
    if not workdir:
        raise RuntimeError(
            f"No compose working_dir label found for container {container}"
        )

    return project, workdir


def main(
    base_directory: str, manipulation_services: list[str], timeout: int | None
) -> int:
    _ = base_directory
    errors = 0
    wait_while_manipulation_running(
        manipulation_services, waiting_time=600, timeout=timeout
    )

    unhealthy_container_names = print_bash(
        "container ps --filter label=com.docker.compose.project --filter health=unhealthy --format '{{.Names}}'"
    )
    exited_lines = print_bash(
        "container ps --filter label=com.docker.compose.project --filter status=exited "
        "--format '{{.Names}}\t{{.Status}}'"
    )
    exited_container_names = [
        line.split("\t", 1)[0]
        for line in exited_lines
        if "\t" in line and not line.split("\t", 1)[1].startswith("Exited (0)")
    ]
    failed_containers = unhealthy_container_names + exited_container_names

    for container in failed_containers:
        try:
            project, workdir = get_compose_project_info(container)
        except Exception as e:
            print(f"Error reading compose labels for {container}: {e}")
            errors += 1
            continue

        compose_file_path = os.path.join(workdir, "compose.yml")
        if not os.path.isfile(compose_file_path):
            print(
                f"Error: compose.yml not found at {compose_file_path} for container {container}"
            )
            errors += 1
            continue

        project_path = os.path.dirname(compose_file_path)
        try:
            print(
                "Restarting unhealthy/exited container via project:", compose_file_path
            )
            log_kill("compose restart", project)
            print_bash(compose_cmd("restart", project_path, project))
        except Exception as e:
            if "port is already allocated" in str(e):
                print("Detected port allocation problem. Executing recovery steps...")
                try:
                    log_kill(
                        "port-conflict recovery: compose down + systemctl restart docker",
                        project,
                    )
                    print_bash(compose_cmd("down", project_path, project))
                    print_bash("systemctl restart docker")
                    print_bash(compose_cmd("up -d", project_path, project))
                except Exception as e2:
                    print("Unhandled exception during recovery:", e2)
                    errors += 1
            else:
                print("Unhandled exception during restart:", e)
                errors += 1

    print("Finished restart procedure.")
    return 1 if errors else 0


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="Restart Docker-Compose configurations with exited or unhealthy containers (STRICT label mode)."
    )
    parser.add_argument(
        "--manipulation",
        metavar="SERVICE",
        nargs="*",
        help="Blocking systemd services to wait for (can be specified multiple times).",
    )
    parser.add_argument(
        "--manipulation-string",
        type=str,
        help='Blocking services as a single string (space- or comma-separated), e.g. "svc1 svc2" or "svc1,svc2".',
    )
    parser.add_argument(
        "--timeout",
        type=int,
        default=60,
        help="Maximum time in seconds to wait for manipulation services before continuing. (Default 1min)",
    )
    parser.add_argument(
        "base_directory",
        type=str,
        help="(Unused in STRICT mode) Base directory where Docker Compose configurations are located.",
    )
    args = parser.parse_args()
    services = normalize_services_arg(args.manipulation, args.manipulation_string)
    raise SystemExit(main(args.base_directory, services, args.timeout))