diff --git a/.github/workflows/call-test-workspace.yml b/.github/workflows/call-test-workspace.yml index 50d605ae83..64753dff93 100644 --- a/.github/workflows/call-test-workspace.yml +++ b/.github/workflows/call-test-workspace.yml @@ -168,7 +168,7 @@ jobs: INFINITO_RESCUE_DIAGNOSTICS_DIR="${rescue_dir}" \ python3 utils/diagnostics/container.py \ "${INFINITO_WORKSPACE_SLUG}" "workspace suite failure" || true - bash ./scripts/tests/deploy/utils/rescue_index.sh "${rescue_dir}" + bash ./scripts/tests/deploy/utils/rescue/index.sh "${rescue_dir}" - name: โฌ†๏ธ Upload rescue diagnostics if: always() diff --git a/scripts/tests/deploy/ci/dedicated.sh b/scripts/tests/deploy/ci/dedicated.sh index bf868efc8a..ccb4156ff6 100755 --- a/scripts/tests/deploy/ci/dedicated.sh +++ b/scripts/tests/deploy/ci/dedicated.sh @@ -23,8 +23,8 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" # shellcheck source=/dev/null source <(grep -E '^INFINITO_(PLAYWRIGHT_REPORTS_BASE_DIR|RESCUE_DIAGNOSTICS_DIR|RESCUE_DIAGNOSTICS_BASE|DNS53_SAMPLER_LOG)=' "${REPO_ROOT}/.env") : "${INFINITO_RESCUE_DIAGNOSTICS_BASE:?INFINITO_RESCUE_DIAGNOSTICS_BASE must be set}" -# shellcheck source=scripts/tests/deploy/utils/rescue_window.sh -. "${REPO_ROOT}/scripts/tests/deploy/utils/rescue_window.sh" +# shellcheck source=scripts/tests/deploy/utils/rescue/window.sh +. "${REPO_ROOT}/scripts/tests/deploy/utils/rescue/window.sh" apps="" @@ -108,7 +108,7 @@ cleanup() { timeout --foreground "${rescue_secs}" python3 "${REPO_ROOT}/utils/diagnostics/container.py" \ "${apps}" "ci host at compose post-deploy failure" 2>/dev/null || true # nocheck: shell-or-true -- the collector exits 1 by contract so the caller cannot mistake a snapshot for a passing deploy - bash scripts/tests/deploy/utils/rescue_index.sh "${_rescue_host_dir}" + bash scripts/tests/deploy/utils/rescue/index.sh "${_rescue_host_dir}" echo ">>> Removing stack for distro ${INFINITO_DISTRO} (fresh start for next distro)" "${PYTHON}" -m cli.administration.deploy.development down || true # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error diff --git a/scripts/tests/deploy/swarm/README.md b/scripts/tests/deploy/swarm/README.md index c9e44a808a..20ec661523 100644 --- a/scripts/tests/deploy/swarm/README.md +++ b/scripts/tests/deploy/swarm/README.md @@ -160,7 +160,7 @@ the cluster declaration (image, containers, network, DNS play) in | chaos | `routine/06_drain_worker.sh` | drain the app's worker + force reschedule | | chaos | `routine/07_assert_state.sh` | assert the marker + reachability survived | | teardown | `utils/collect/diagnostics.sh` | write stack/service diagnostics on failure, one file per topic under the rescue dir | -| teardown | `../utils/rescue_index.sh` | list the collected rescue tree so a reader knows what the artifact holds | +| teardown | `../utils/rescue/index.sh` | list the collected rescue tree so a reader knows what the artifact holds | | teardown | `utils/collect/playwright_reports.sh` | pull Playwright reports from the manager into `/tmp/playwright-artifacts//` | | teardown | `utils/collect/topology_summary.sh` | render the node list + service placement into `$GITHUB_STEP_SUMMARY` | | teardown | `utils/clean/teardown.sh` | kill the nodes + remove the lab network | diff --git a/scripts/tests/deploy/swarm/routine/00_one.sh b/scripts/tests/deploy/swarm/routine/00_one.sh index 0cec6f0fc7..e8408dea95 100755 --- a/scripts/tests/deploy/swarm/routine/00_one.sh +++ b/scripts/tests/deploy/swarm/routine/00_one.sh @@ -111,7 +111,7 @@ collect_and_teardown() { run_within_deadline 900 "$((collect_floor_seconds + index_floor_seconds * 2 + teardown_floor_seconds))" \ bash "${SCRIPT_DIR}/../utils/collect/diagnostics.sh" || true # nocheck: shell-or-true -- best-effort diagnostics + teardown in the EXIT trap run_within_deadline 300 "$((collect_floor_seconds + index_floor_seconds + teardown_floor_seconds))" \ - bash "${REPO_ROOT}/scripts/tests/deploy/utils/rescue_index.sh" "${rescue_dir}" || true # nocheck: shell-or-true -- best-effort diagnostics + teardown in the EXIT trap + bash "${REPO_ROOT}/scripts/tests/deploy/utils/rescue/index.sh" "${rescue_dir}" || true # nocheck: shell-or-true -- best-effort diagnostics + teardown in the EXIT trap fi run_within_deadline 900 "$((index_floor_seconds + teardown_floor_seconds))" \ diff --git a/scripts/tests/deploy/utils/rescue_index.sh b/scripts/tests/deploy/utils/rescue/index.sh similarity index 97% rename from scripts/tests/deploy/utils/rescue_index.sh rename to scripts/tests/deploy/utils/rescue/index.sh index 46112b9687..7df4110fd3 100755 --- a/scripts/tests/deploy/utils/rescue_index.sh +++ b/scripts/tests/deploy/utils/rescue/index.sh @@ -9,7 +9,7 @@ set -euo pipefail trap 'exit 0' EXIT -DIR="${1:?usage: rescue_index.sh DIR [LIMIT]}" +DIR="${1:?usage: index.sh DIR [LIMIT]}" LIMIT="${2:-400}" [ -d "${DIR}" ] || exit 0 diff --git a/scripts/tests/deploy/utils/rescue_window.sh b/scripts/tests/deploy/utils/rescue/window.sh similarity index 100% rename from scripts/tests/deploy/utils/rescue_window.sh rename to scripts/tests/deploy/utils/rescue/window.sh diff --git a/tests/unit/python/scripts/tests/deploy/utils/rescue/__init__.py b/tests/unit/python/scripts/tests/deploy/utils/rescue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/python/scripts/tests/deploy/utils/test_rescue_index.py b/tests/unit/python/scripts/tests/deploy/utils/rescue/test_index.py similarity index 97% rename from tests/unit/python/scripts/tests/deploy/utils/test_rescue_index.py rename to tests/unit/python/scripts/tests/deploy/utils/rescue/test_index.py index 343e110204..1aa30334a9 100644 --- a/tests/unit/python/scripts/tests/deploy/utils/test_rescue_index.py +++ b/tests/unit/python/scripts/tests/deploy/utils/rescue/test_index.py @@ -1,6 +1,6 @@ """The caller contract of the rescue-artifact index. -``rescue_index.sh`` runs inside cleanup handlers and in a GitHub step under +``rescue/index.sh`` runs inside cleanup handlers and in a GitHub step under ``bash -e`` with no fallback, so its exit status decides whether the caller finishes tearing down: a fatal usage error and a ``find`` that rejects ``-printf`` both have to leave the caller at rc 0, and the latter has to stay @@ -21,7 +21,7 @@ from pathlib import Path from utils.cache.files import PROJECT_ROOT -INDEX = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "utils" / "rescue_index.sh" +INDEX = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "utils" / "rescue" / "index.sh" FIND_WITHOUT_PRINTF = '#!/bin/sh\necho "find: unrecognized: -printf" >&2\nexit 1\n' GLYPHS = ("๐Ÿ“", "๐Ÿ“„", "๐Ÿ“œ", "๐Ÿงพ") FIND_CUT_SHORT = ( diff --git a/tests/unit/python/scripts/tests/deploy/utils/test_rescue_window.py b/tests/unit/python/scripts/tests/deploy/utils/rescue/test_window.py similarity index 95% rename from tests/unit/python/scripts/tests/deploy/utils/test_rescue_window.py rename to tests/unit/python/scripts/tests/deploy/utils/rescue/test_window.py index f6f57e7820..931972e67f 100644 --- a/tests/unit/python/scripts/tests/deploy/utils/test_rescue_window.py +++ b/tests/unit/python/scripts/tests/deploy/utils/rescue/test_window.py @@ -13,12 +13,13 @@ from pathlib import Path from utils.cache.files import PROJECT_ROOT, read_text DEPLOY = PROJECT_ROOT / "scripts" / "tests" / "deploy" -HELPER = DEPLOY / "utils" / "rescue_window.sh" -INDEX = DEPLOY / "utils" / "rescue_index.sh" +HELPER = DEPLOY / "utils" / "rescue" / "window.sh" +INDEX = DEPLOY / "utils" / "rescue" / "index.sh" DEDICATED = DEPLOY / "ci" / "dedicated.sh" COLLECTOR = PROJECT_ROOT / "utils" / "diagnostics" / "container.py" DISTRO_PREFIX = "INFINITO_CI_DISTRO_" # nocheck: prefix the drills strip, not a key GRACE = 8 +KILL_BUDGET = 3 SKIPPED = b"[skipped: past RESCUE_DEADLINE]\n" WINDOW = """\ @@ -114,7 +115,7 @@ class RescueWindowTests(unittest.TestCase): class DedicatedWiringTests(unittest.TestCase): def test_every_capture_runs_inside_the_window(self) -> None: text = read_text(str(DEDICATED)) - self.assertIn("utils/rescue_window.sh", text) + self.assertIn("utils/rescue/window.sh", text) self.assertIn("rescue_window 1500", text) self.assertIn("rescue_window 900", text) commands = text.replace("\\\n", " ").replace('"', "").splitlines() @@ -172,7 +173,7 @@ class BudgetKillDrillTests(unittest.TestCase): def test_a_budget_kill_copies_a_partial_snapshot_out_inside_the_grace(self) -> None: proc, kill_epoch, rescued, files = self._drill( - "sleep", "1000", budget=1, journal_seconds="30", epochs=True + "sleep", "1000", budget=KILL_BUDGET, journal_seconds="30", epochs=True ) self.assertEqual(proc.returncode, 124, proc.stdout) self.assertIn("Budget kill:", proc.stdout) @@ -185,7 +186,7 @@ class BudgetKillDrillTests(unittest.TestCase): def test_without_the_window_the_kill_lands_before_the_copy(self) -> None: proc, _kill_epoch, rescued, files = self._drill( - "sleep", "1000", budget=1, journal_seconds="30", epochs=False + "sleep", "1000", budget=KILL_BUDGET, journal_seconds="30", epochs=False ) self.assertEqual(proc.returncode, 137, proc.stdout) self.assertIsNone(rescued) diff --git a/utils/diagnostics/README.md b/utils/diagnostics/README.md index 9bd238cb9c..ce22b03a41 100644 --- a/utils/diagnostics/README.md +++ b/utils/diagnostics/README.md @@ -11,7 +11,7 @@ An optional `RESCUE_DEADLINE` epoch caps every command on every level; once it has passed, each remaining capture is written as a `[skipped: past RESCUE_DEADLINE]` placeholder. The compose trap in [dedicated.sh](../../scripts/tests/deploy/ci/dedicated.sh) sets it through -[rescue_window.sh](../../scripts/tests/deploy/utils/rescue_window.sh) only when +[rescue/window.sh](../../scripts/tests/deploy/utils/rescue/window.sh) only when the distro budget's SIGTERM has already arrived. ```mermaid