diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5da654f330..fc98c21db5 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -237,6 +237,7 @@ reproduces by re-running the same sweep: The vulnerability engine is the odometer digit above distro and filesystem, and its glyph follows the filesystem glyph in the label (`🐳🧅🌀🦓🔺`). The deploy-time scan of `sys-ctl-hlth-runtime` then runs Syft and that engine only, plus Grype over the source-built runtimes Syft finds. The `vulnerability-db` job of [call-test-deploy.yml](call-test-deploy.yml) downloads the Trivy, Grype and OSV databases once per day into the Actions cache; each deploy job restores the Grype database and the drawn engine's into `build/vulnerability-db`, which the stack and every swarm node mount read-only, and fails when one is missing, so no deploy job downloads a database itself. +Under act these steps are skipped: the deploy draws its engine and refreshes the shared host directory `INFINITO_VULNERABILITY_DB_HOST_PATH`, which [workflow.sh](../../scripts/tests/deploy/act/workflow.sh) binds into the job container at the same path. `cron-images-mirror-all.yml` writes the findings of its `scan` job per image and per CVE into the job summary and the `vulnerability-findings` artifact; dispatched with `issues: false` it is a dry run that leaves the issues untouched. A role offers at most two modes in practice — swarm needs its own stack, host diff --git a/.github/workflows/call-test-deploy.yml b/.github/workflows/call-test-deploy.yml index 0ffb083d83..a41170f6f4 100644 --- a/.github/workflows/call-test-deploy.yml +++ b/.github/workflows/call-test-deploy.yml @@ -337,7 +337,6 @@ jobs: INFINITO_DOCKER_FILESYSTEM_ALLOWED: ${{ inputs.filesystem }} INFINITO_DOCKER_FILESYSTEM_PICK: ${{ matrix.filesystem }} INFINITO_DOCKER_FILESYSTEM_ENFORCED: ${{ matrix.enforce_filesystem }} - INFINITO_VULNERABILITY_ENGINE: ${{ matrix.engine }} RESCHED_TIMEOUT: "120" SCRIPTS: scripts/tests/deploy/swarm/routine @@ -378,6 +377,18 @@ jobs: key: vulnerability-db-${{ matrix.engine }}-${{ needs.vulnerability-db.outputs.day }} fail-on-cache-miss: true + - name: "🔬 Pin the deploy to the restored databases" + if: ${{ !env.ACT }} + shell: bash + env: + ENGINE: ${{ matrix.engine }} + run: | + set -euo pipefail + { + echo "INFINITO_VULNERABILITY_ENGINE=${ENGINE}" + echo "INFINITO_VULNERABILITY_DB_HOST_PATH=${GITHUB_WORKSPACE}/build/vulnerability-db" + } >> "${GITHUB_ENV}" + - name: 💾 Enlarge swap space if: ${{ !env.ACT }} shell: bash @@ -438,7 +449,6 @@ jobs: disable: ${{ format('{0} {1}', matrix.disable, inputs.disable) }} INFINITO_SWARM_VPN: ${{ matrix.vpn }} INFINITO_SWARM_MESHES: ${{ inputs.meshes }} - INFINITO_VULNERABILITY_DB_HOST_PATH: ${{ github.workspace }}/build/vulnerability-db run: ./scripts/tests/deploy/distros.sh "${{ env.SCRIPTS }}/00_one.sh" 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log" - name: 🚀 Deploy app @@ -450,7 +460,6 @@ jobs: variant: ${{ matrix.variant }} disable: ${{ format('{0} {1}', matrix.disable, inputs.disable) }} ANSIBLE_LOG_PATH: /tmp/ansible-${{ matrix.apps }}.log - INFINITO_VULNERABILITY_DB_HOST_PATH: ${{ github.workspace }}/build/vulnerability-db run: ./scripts/tests/deploy/ci/all.sh 2>&1 | tee "/tmp/ansible-${{ matrix.apps }}.log" - name: 🚫 Fail on deprecation warnings diff --git a/.github/workflows/call-test-workspace.yml b/.github/workflows/call-test-workspace.yml index 55132afadc..e575af4d75 100644 --- a/.github/workflows/call-test-workspace.yml +++ b/.github/workflows/call-test-workspace.yml @@ -117,6 +117,12 @@ jobs: run: | set -euo pipefail HOST_WORKDIR="$(pwd -P)" + if [[ "${ACT:-}" == "true" ]]; then + db_host_path="${INFINITO_VULNERABILITY_DB_HOST_PATH:?scripts/tests/deploy/act/workflow.sh passes the shared host directory}" + else + db_host_path="${HOST_WORKDIR}/build/vulnerability-db" + fi + mkdir -p "${db_host_path}" docker run --rm \ -e GITHUB_ACTIONS=true \ @@ -139,9 +145,10 @@ jobs: -e INFINITO_CACHE_PACKAGE_HOST_PATH \ -e INFINITO_GHCR_MIRROR \ -e INFINITO_GHCR_MIRROR_PREFIX \ - -e INFINITO_VULNERABILITY_DB_HOST_PATH="${HOST_WORKDIR}/build/vulnerability-db" \ + -e INFINITO_VULNERABILITY_DB_HOST_PATH="${db_host_path}" \ --network host \ -v "${HOST_WORKDIR}:${HOST_WORKDIR}" \ + -v "${db_host_path}:${db_host_path}" \ -v "/mnt/docker:/var/lib/docker" \ -v "/var/run/docker.sock:/var/run/docker.sock" \ -w "${HOST_WORKDIR}" \ diff --git a/cli/administration/deploy/development/compose.py b/cli/administration/deploy/development/compose.py index 6167151b12..e6a3178d36 100644 --- a/cli/administration/deploy/development/compose.py +++ b/cli/administration/deploy/development/compose.py @@ -19,10 +19,6 @@ INIT_IMAGE_KEY = "INFINITO_CACHE_PACKAGE_FRONTEND_INIT_IMAGE" def prepare_vulnerability_db(path: Path) -> None: """Create the shared vulnerability database directory as the calling user. - Docker would create a missing bind source as root, and a deploy could then - not refresh the databases in it. A directory this user may not create is - left to Docker: only a deploy writes it, and that deploy fails loudly. - Args: path: ``INFINITO_VULNERABILITY_DB_HOST_PATH``. """ @@ -190,7 +186,13 @@ class Compose: ) def _prepare_vulnerability_db(self, env: dict[str, str]) -> None: - prepare_vulnerability_db(Path(env["INFINITO_VULNERABILITY_DB_HOST_PATH"])) + path = env.get("INFINITO_VULNERABILITY_DB_HOST_PATH") + if not path: + raise SystemExit( + "INFINITO_VULNERABILITY_DB_HOST_PATH is not set; " + "run 'make dotenv' to generate the .env single source of truth" + ) + prepare_vulnerability_db(Path(path)) def up(self, *, run_entry_init: bool = True) -> None: print(">>> Rendering CoreDNS Corefile from template") diff --git a/docs/contributing/environment/cache.md b/docs/contributing/environment/cache.md index 417a1456e9..40b36fb1a8 100644 --- a/docs/contributing/environment/cache.md +++ b/docs/contributing/environment/cache.md @@ -113,7 +113,7 @@ Cache state persists under `/var/cache/infinito/core/cache/`. Paths are configur ## Vulnerability Databases 🔬 -Every development stack and every swarm lab node mounts `INFINITO_VULNERABILITY_DB_HOST_PATH` read-only at `INFINITO_VULNERABILITY_DB_DIR`. A local deploy draws one engine and refreshes that engine's and Grype's database there once a day, under a host-wide lock, before the deploy starts; the stacks never download one. A CI deploy job points the variable at the databases it restored from the Actions cache instead. See [sys-ctl-hlth-runtime](../../../roles/sys-ctl-hlth-runtime/README.md). +Every development stack and every swarm lab node mounts `INFINITO_VULNERABILITY_DB_HOST_PATH` read-only at `INFINITO_VULNERABILITY_DB_DIR`. A local deploy draws one engine and refreshes that engine's and Grype's database there once a day, under a lock file inside that directory, before the deploy starts; the stacks never download one. An act run binds the directory into its job container at the same path, and a refresh as root hands its files to the directory's owner. A CI deploy job points the variable at the databases it restored from the Actions cache instead. See [sys-ctl-hlth-runtime](../../../roles/sys-ctl-hlth-runtime/README.md). ## Adding a New Upstream ➕ diff --git a/scripts/tests/deploy/act/README.md b/scripts/tests/deploy/act/README.md index 537daa51a1..ee3794a2ed 100644 --- a/scripts/tests/deploy/act/README.md +++ b/scripts/tests/deploy/act/README.md @@ -9,4 +9,4 @@ For the canonical Make target index that invokes these helpers, see [make.md](.. | Command | What it does | Notes | |---|---|---| -| `workflow.sh` | Runs any workflow file through `act`. | Supports custom job, matrix, container, network, and image settings. Invoked via `make act-workflow ACT_WORKFLOW=`. | +| `workflow.sh` | Runs any workflow file through `act`. | Supports custom job, matrix, container, network, and image settings. Binds `INFINITO_VULNERABILITY_DB_HOST_PATH` into the job container at the same path. Invoked via `make act-workflow ACT_WORKFLOW=`. | diff --git a/scripts/tests/deploy/act/workflow.sh b/scripts/tests/deploy/act/workflow.sh index 2f39f599ae..8464d3b40e 100755 --- a/scripts/tests/deploy/act/workflow.sh +++ b/scripts/tests/deploy/act/workflow.sh @@ -14,6 +14,7 @@ set -euo pipefail : "${ACT_BIND:=false}" : "${ACT_INPUTS:=}" : "${ACT_ENV:=}" +: "${INFINITO_VULNERABILITY_DB_HOST_PATH:?INFINITO_VULNERABILITY_DB_HOST_PATH is not set; run 'make dotenv' to generate the .env single source of truth}" if [[ "${ACT_PLATFORM_IMAGE}" == local/act-runner-fixed:latest ]] && ! docker image inspect "${ACT_PLATFORM_IMAGE}" >/dev/null 2>&1; then @@ -83,9 +84,9 @@ if [[ -n "${ACT_ENV}" ]]; then fi done fi -if [[ -n "${ACT_CONTAINER_OPTIONS}" ]]; then - cmd+=(--container-options "${ACT_CONTAINER_OPTIONS}") -fi +mkdir -p "${INFINITO_VULNERABILITY_DB_HOST_PATH}" +cmd+=(--container-options "${ACT_CONTAINER_OPTIONS} --volume ${INFINITO_VULNERABILITY_DB_HOST_PATH}:${INFINITO_VULNERABILITY_DB_HOST_PATH}") +cmd+=(--env "INFINITO_VULNERABILITY_DB_HOST_PATH=${INFINITO_VULNERABILITY_DB_HOST_PATH}") if [[ -n "${ACT_NETWORK}" ]]; then cmd+=(--network "${ACT_NETWORK}") fi diff --git a/tests/lint/repository/test_deploy_vulnerability_databases.py b/tests/lint/repository/test_deploy_vulnerability_databases.py new file mode 100644 index 0000000000..62bfe0d725 --- /dev/null +++ b/tests/lint/repository/test_deploy_vulnerability_databases.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from tests.utils import PROJECT_ROOT +from utils.cache.files import read_text +from utils.cache.yaml import load_yaml_any + +WORKFLOWS = PROJECT_ROOT / ".github" / "workflows" +ENGINE = "INFINITO_VULNERABILITY_ENGINE" +DB_HOST_PATH = "INFINITO_VULNERABILITY_DB_HOST_PATH" +GITHUB_ONLY = "${{ !env.ACT }}" + + +def steps(workflow: str, job: str) -> list[dict]: + return load_yaml_any(str(WORKFLOWS / workflow))["jobs"][job]["steps"] + + +def bash(script: str, env: dict[str, str], cwd: str) -> subprocess.CompletedProcess: + clean = { + k: v + for k, v in os.environ.items() + if k not in ("BASH_ENV", "ACT", DB_HOST_PATH) + } + return subprocess.run( + ["bash", "-c", script], + env={**clean, **env}, + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +class TestDeployJob(unittest.TestCase): + def setUp(self) -> None: + self.job = load_yaml_any(str(WORKFLOWS / "call-test-deploy.yml"))["jobs"][ + "deploy" + ] + self.pins = [ + step for step in self.job["steps"] if f"{ENGINE}=" in step.get("run", "") + ] + + def test_no_env_block_pins_the_engine_or_the_path(self) -> None: + blocks = [self.job["env"], *(step.get("env", {}) for step in self.job["steps"])] + self.assertEqual( + [block for block in blocks if {ENGINE, DB_HOST_PATH} & set(block)], [] + ) + + def test_only_a_github_run_pins_the_restored_databases(self) -> None: + self.assertEqual([step["if"] for step in self.pins], [GITHUB_ONLY]) + + def test_the_pin_comes_before_every_deploy(self) -> None: + names = [step["name"] for step in self.job["steps"]] + deploys = [index for index, name in enumerate(names) if name.startswith("🚀")] + self.assertTrue(deploys) + self.assertLess(names.index(self.pins[0]["name"]), min(deploys)) + + def test_a_github_run_pins_the_row_engine_and_the_restored_path(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + github_env = Path(tmp) / "github_env" + done = bash( + self.pins[0]["run"], + { + "ENGINE": "osv", + "GITHUB_WORKSPACE": "/ws", + "GITHUB_ENV": str(github_env), + }, + tmp, + ) + self.assertEqual(done.returncode, 0, done.stderr) + lines = read_text(str(github_env)).splitlines() + self.assertEqual( + lines, [f"{ENGINE}=osv", f"{DB_HOST_PATH}=/ws/build/vulnerability-db"] + ) + + +class TestWorkspaceJob(unittest.TestCase): + def setUp(self) -> None: + script = next( + step["run"] + for step in steps("call-test-workspace.yml", "test-workspace") + if "00_orchestrator.sh" in step.get("run", "") + ) + self.script = script + self.prelude = ( + script[: script.index("docker run")] + 'printf "%s" "${db_host_path}"\n' + ) + + def _db_host_path( + self, env: dict[str, str] + ) -> tuple[subprocess.CompletedProcess, str]: + with tempfile.TemporaryDirectory() as tmp: + workdir = Path(tmp).resolve() + env = { + key: value.replace("{tmp}", str(workdir)) for key, value in env.items() + } + done = bash(self.prelude, env, str(workdir)) + return done, done.stdout.replace(str(workdir), "{tmp}") + + def test_an_act_run_shares_the_host_directory_of_the_wrapper(self) -> None: + done, path = self._db_host_path({"ACT": "true", DB_HOST_PATH: "{tmp}/shared"}) + self.assertEqual((done.returncode, path), (0, "{tmp}/shared")) + + def test_a_github_run_reads_the_databases_of_its_checkout(self) -> None: + done, path = self._db_host_path({}) + self.assertEqual((done.returncode, path), (0, "{tmp}/build/vulnerability-db")) + + def test_an_act_run_without_the_wrapper_path_fails(self) -> None: + done, _path = self._db_host_path({"ACT": "true"}) + self.assertNotEqual(done.returncode, 0) + + def test_the_container_sees_the_databases_at_the_host_path(self) -> None: + self.assertIn('-v "${db_host_path}:${db_host_path}"', self.script) + self.assertIn(f'-e {DB_HOST_PATH}="${{db_host_path}}"', self.script) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/cli/administration/deploy/development/test_compose.py b/tests/unit/python/cli/administration/deploy/development/test_compose.py index 7741007a03..afd55ccc71 100644 --- a/tests/unit/python/cli/administration/deploy/development/test_compose.py +++ b/tests/unit/python/cli/administration/deploy/development/test_compose.py @@ -331,6 +331,11 @@ class TestPrepareVulnerabilityDb(unittest.TestCase): ) prepare.assert_called_once_with(Path("/srv/db")) + def test_a_stale_dotenv_fails_with_the_regeneration_hint(self) -> None: + compose = Compose(repo_root=Path("/tmp/infinito-nexus"), distro="arch") + with self.assertRaisesRegex(SystemExit, "make dotenv"): + compose._prepare_vulnerability_db({}) + if __name__ == "__main__": # pragma: no cover unittest.main() diff --git a/tests/unit/python/scripts/tests/deploy/act/test_workflow.py b/tests/unit/python/scripts/tests/deploy/act/test_workflow.py new file mode 100644 index 0000000000..6d007cfe14 --- /dev/null +++ b/tests/unit/python/scripts/tests/deploy/act/test_workflow.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +from utils.cache.files import PROJECT_ROOT + +SCRIPT = PROJECT_ROOT / "scripts" / "tests" / "deploy" / "act" / "workflow.sh" +ACT_STUB = '#!/usr/bin/env bash\nprintf "%s\\n" "$@" >"${STUB_LOG}"\n' +DB_ENV = "INFINITO_VULNERABILITY_DB_HOST_PATH" + + +def run_act( + db_host_path: str | None, +) -> tuple[subprocess.CompletedProcess, list[str], Path, bool]: + """Run the wrapper against an ``act`` stub that records its argv. + + Args: + db_host_path: value of ``INFINITO_VULNERABILITY_DB_HOST_PATH`` below a + scratch directory, or None to leave it unset. + + Returns: + The finished process, the argv act received, the shared database + directory the run was given, and whether it existed afterwards. + """ + with tempfile.TemporaryDirectory() as tmp: + stub = Path(tmp) / "act" + stub.write_text(ACT_STUB) + stub.chmod(0o755) + log = Path(tmp) / "act.log" + log.touch() + workflow = Path(tmp) / "call-test-deploy.yml" + workflow.write_text("on: workflow_dispatch\njobs: {}\n") + env = { + k: v + for k, v in os.environ.items() + if k not in ("BASH_ENV", DB_ENV, "ACT_CONTAINER_OPTIONS") + } + env.update( + PATH=f"{tmp}{os.pathsep}{os.environ['PATH']}", + STUB_LOG=str(log), + ACT_WORKFLOW=str(workflow), + ACT_ACTION_CACHE_PATH=str(Path(tmp) / "actcache"), + INFINITO_CONTAINER="infinito-test", + ACT_PLATFORM_IMAGE="catthehacker/ubuntu:act-latest", + ) + shared = Path(tmp) / "cache" / (db_host_path or "unset") + if db_host_path is not None: + env[DB_ENV] = str(shared) + done = subprocess.run( + ["bash", str(SCRIPT)], + env=env, + capture_output=True, + text=True, + check=False, + ) + argv = log.read_text().splitlines() # nocheck: cache-read stub call log + return done, argv, shared, shared.is_dir() + + +def option(argv: list[str], flag: str) -> list[str]: + return [argv[index + 1] for index, value in enumerate(argv[:-1]) if value == flag] + + +class TestVulnerabilityDatabaseBind(unittest.TestCase): + def test_the_shared_databases_sit_at_the_host_path_in_the_job_container( + self, + ) -> None: + done, argv, shared, _created = run_act("vulnerability-db") + self.assertEqual(done.returncode, 0, done.stderr) + self.assertIn( + f"--volume {shared}:{shared}", option(argv, "--container-options")[0] + ) + + def test_the_job_steps_receive_the_host_path(self) -> None: + _done, argv, shared, _created = run_act("vulnerability-db") + self.assertIn(f"{DB_ENV}={shared}", option(argv, "--env")) + + def test_the_directory_exists_before_act_binds_it(self) -> None: + _done, _argv, _shared, created = run_act("vulnerability-db") + self.assertTrue(created) + + def test_the_job_container_keeps_the_host_daemon_socket(self) -> None: + _done, argv, _shared, _created = run_act("vulnerability-db") + self.assertNotIn("--container-daemon-socket", argv) + + def test_the_default_options_survive(self) -> None: + _done, argv, _shared, _created = run_act("vulnerability-db") + self.assertTrue( + option(argv, "--container-options")[0].startswith( + "--privileged --label infinito.worktree=infinito-test " + ) + ) + + def test_an_unset_host_path_starts_no_act_run(self) -> None: + done, argv, _shared, created = run_act(None) + self.assertNotEqual(done.returncode, 0) + self.assertEqual((argv, created), ([], False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/utils/security/test_database.py b/tests/unit/python/utils/security/test_database.py index f0f9cde27a..547cc19252 100644 --- a/tests/unit/python/utils/security/test_database.py +++ b/tests/unit/python/utils/security/test_database.py @@ -96,29 +96,79 @@ class TestStale(_Root): class TestRefresh(_Root): + def _refresh(self, engines, fetched=()): + with mock.patch.object( + database, "fetch_databases", return_value=list(fetched) + ) as fetch: + errors = database.refresh( + self.root, engines, lambda argv: None, lambda u, d: None + ) + return errors, fetch + def test_only_stale_engines_are_downloaded(self): _touch(self.root / "grype" / "6" / "vulnerability.db", 60) - with ( - mock.patch.object(database, "LOCK_FILE", self.root / "lock"), - mock.patch.object(database, "fetch_databases", return_value=[]) as fetch, - ): - database.refresh( - self.root, ["trivy", "grype"], lambda argv: None, lambda u, d: None - ) + _errors, fetch = self._refresh(["trivy", "grype"]) self.assertEqual(fetch.call_args.args[2], ["trivy"]) def test_fresh_databases_download_nothing(self): _touch(self.root / "grype" / "6" / "vulnerability.db", 60) - with ( - mock.patch.object(database, "LOCK_FILE", self.root / "lock"), - mock.patch.object(database, "fetch_databases") as fetch, - ): - errors = database.refresh( - self.root, ["grype"], lambda argv: None, lambda u, d: None - ) + errors, fetch = self._refresh(["grype"]) self.assertEqual(errors, []) fetch.assert_not_called() + def test_the_lock_lives_in_the_shared_directory(self): + self._refresh(["grype"]) + self.assertTrue((self.root / database.LOCK_NAME).is_file()) + + def test_a_missing_shared_directory_is_created(self): + self.root = self.root / "vulnerability-db" + self._refresh(["grype"]) + self.assertTrue(self.root.is_dir()) + + def test_a_failed_download_is_returned(self): + errors, _fetch = self._refresh(["grype"], ["grype database: exited 1"]) + self.assertEqual(errors, ["grype database: exited 1"]) + + +class TestHandOver(_Root): + def _hand_over(self, euid): + _touch(self.root / "osv" / "osv-scalibr" / "Ubuntu" / "all.zip", 60) + owner = self.root.stat() + with ( + mock.patch.object(database.os, "geteuid", return_value=euid), + mock.patch.object(database.os, "chown") as chown, + ): + database.hand_over(self.root, [self.root / "osv"]) + return chown, (owner.st_uid, owner.st_gid) + + def test_a_root_refresh_gives_every_file_to_the_directory_owner(self): + chown, owner = self._hand_over(0) + self.assertEqual( + {(Path(call.args[0]), call.args[1:]) for call in chown.call_args_list}, + { + (self.root / "osv", owner), + (self.root / "osv" / "osv-scalibr", owner), + (self.root / "osv" / "osv-scalibr" / "Ubuntu", owner), + (self.root / "osv" / "osv-scalibr" / "Ubuntu" / "all.zip", owner), + }, + ) + + def test_an_unprivileged_refresh_changes_no_owner(self): + chown, _owner = self._hand_over(1000) + chown.assert_not_called() + + def test_a_root_refresh_that_wrote_no_database_hands_over_the_lock(self): + with ( + mock.patch.object(database.os, "geteuid", return_value=0), + mock.patch.object(database.os, "chown") as chown, + mock.patch.object(database, "fetch_databases", return_value=[]), + ): + database.refresh(self.root, ["grype"], lambda argv: None, lambda u, d: None) + self.assertEqual( + [Path(call.args[0]) for call in chown.call_args_list], + [self.root / database.LOCK_NAME], + ) + if __name__ == "__main__": main() diff --git a/utils/security/database.py b/utils/security/database.py index 79d884fb5e..d5b09a6c27 100644 --- a/utils/security/database.py +++ b/utils/security/database.py @@ -1,15 +1,10 @@ -"""Download the vulnerability databases into one shared directory. - -The CI ``vulnerability-db`` job fills the Actions cache with it, and a local -deploy refreshes the host directory every stack mounts read-only, so no -stack and no node downloads a database itself. -""" +"""Download the vulnerability databases into one shared directory.""" from __future__ import annotations import fcntl +import os import shutil -import tempfile import time import urllib.parse import urllib.request @@ -25,7 +20,7 @@ if TYPE_CHECKING: OSV_BUCKET = "https://osv-vulnerabilities.storage.googleapis.com" DOWNLOAD_TIMEOUT_SECONDS = 600 MAX_AGE_SECONDS = 24 * 3600 -LOCK_FILE = Path(tempfile.gettempdir()) / "infinito-vulnerability-db.lock" +LOCK_NAME = ".lock" def osv_url(ecosystem: str) -> str: @@ -104,6 +99,25 @@ def stale(p: ModuleType, root: Path, engine: str, now: float) -> bool: ) +def hand_over(root: Path, paths: Sequence[Path]) -> None: + """Give *paths* and everything below them to the owner of *root*. + + Args: + root: the shared database directory. + paths: what this process may have written below *root*; a path it + did not create is skipped. + """ + if os.geteuid() != 0: + return + owner = root.stat() + for path in (path for path in paths if path.exists()): + os.chown(path, owner.st_uid, owner.st_gid) + tree = os.walk(path) # nocheck: project-walk - not the repository + for directory, dirs, files in tree: + for name in (*dirs, *files): + os.chown(Path(directory) / name, owner.st_uid, owner.st_gid) + + def refresh( root: Path, engines: Sequence[str], @@ -112,14 +126,15 @@ def refresh( ) -> list[str]: """Download every database of *engines* that is missing or stale. - The host-wide lock keeps two deploys from writing the shared directory - at once. - Returns: The error lines of the downloads that failed. """ p = pipeline() - with LOCK_FILE.open("w") as lock: + root.mkdir(parents=True, exist_ok=True) + lock_path = root / LOCK_NAME + with lock_path.open("a") as lock: fcntl.flock(lock, fcntl.LOCK_EX) outdated = [engine for engine in engines if stale(p, root, engine, time.time())] - return fetch_databases(p, root, outdated, runner, fetch) if outdated else [] + errors = fetch_databases(p, root, outdated, runner, fetch) if outdated else [] + hand_over(root, [lock_path, *(root / engine for engine in outdated)]) + return errors