fix(web-app-moodle): let a second bootstrap stand down instead of swapping again

Job 116229890655 died in 03_ownership.yml after 69 seconds with 5089 chmod errors, every one of them under .bootstrap.stage, and the walk ended on 'find: failed to read file names from file system at or below /var/www/html: Stale file handle'. A second moodle-entrypoint.sh bootstrap completed inside that window and tore the staging tree away under the running find. The doubled phpspreadsheet path segment in the output is incidental.

0c748f74ed added the staging directory so a failed copy leaves the old tree in place, and guarded two instances on different releases through the version check. Two instances on the SAME release still race, because the lock lives on the code volume and does not serialize across NFS clients. moodle_bootstrap_code_dir now re-reads the sentinel after its copy and stands down when another container has finished the same bootstrap.

The three code-dir walks prune the staging directory, so a teardown mid-walk no longer fails them, and they batch through -exec + instead of one process per entry. MOODLE_BOOTSTRAP_STAGE moves into vars/main.yml and reaches the image as an ARG and ENV, so every service built from it, cron included, resolves the same path.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

roles/web-app-moodle/files/Dockerfile

@@ -4,7 +4,7 @@
 # compose.yml.j2, which derives them from roles/web-app-moodle/vars/main.yml.
 # That vars file is the single source of truth for:
 #   - container paths    (MOODLE_CODE_DIR, MOODLE_DATA_DIR, MOODLE_SOURCE_DIR,
-#                         MOODLE_ENTRYPOINT_PATH)
+#                         MOODLE_ENTRYPOINT_PATH, MOODLE_BOOTSTRAP_STAGE)
 #   - build-asset names  (MOODLE_BUILD_DIR, MOODLE_BUILD_FILES.*)
 #
 # The COPY src paths and the ENTRYPOINT JSON-array literal in this
@@ -22,6 +22,7 @@ ARG MOODLE_CODE_DIR
 ARG MOODLE_DATA_DIR
 ARG MOODLE_LOCAL_CACHE_DIR
 ARG MOODLE_SOURCE_DIR
+ARG MOODLE_BOOTSTRAP_STAGE
 ARG MOODLE_RUNTIME_USER
 ARG MOODLE_OIDC_ENABLED
 ARG MOODLE_OIDC_PLUGIN_RELPATH
@@ -43,6 +44,7 @@ ENV MOODLE_CODE_DIR=${MOODLE_CODE_DIR} \
     MOODLE_DATA_DIR=${MOODLE_DATA_DIR} \
     MOODLE_LOCAL_CACHE_DIR=${MOODLE_LOCAL_CACHE_DIR} \
     MOODLE_SOURCE_DIR=${MOODLE_SOURCE_DIR} \
+    MOODLE_BOOTSTRAP_STAGE=${MOODLE_BOOTSTRAP_STAGE} \
     MOODLE_RUNTIME_USER=${MOODLE_RUNTIME_USER} \
     MOODLE_OIDC_PLUGIN_RELPATH=${MOODLE_OIDC_PLUGIN_RELPATH} \
     MOODLE_MCP_PLUGIN_RELPATH=${MOODLE_MCP_PLUGIN_RELPATH} \

roles/web-app-moodle/files/build/moodle-entrypoint.sh

@@ -6,19 +6,23 @@
 # /proc/self/fd/2; workers fork to ${MOODLE_RUNTIME_USER} per the pool).
 # All paths come from env vars exported by the Dockerfile (single
 # source of truth: roles/web-app-moodle/vars/main.yml).
+#
+# The lock sits on the code volume and does not serialize the replicas and
+# cron across NFS clients, so the sentinel is re-read after the copy: a
+# second swap would wipe the tree the deploy has already installed into.
 set -euo pipefail
 
 : "${MOODLE_CODE_DIR:?required}"
 : "${MOODLE_DATA_DIR:?required}"
 : "${MOODLE_LOCAL_CACHE_DIR:?required}"
 : "${MOODLE_SOURCE_DIR:?required}"
+: "${MOODLE_BOOTSTRAP_STAGE:?required}"
 : "${MOODLE_RUNTIME_USER:?required}"
 : "${MOODLE_VERSION_FILE:?required}"
 : "${MOODLE_RELEASE:?required}"
 
 MOODLE_BOOTSTRAP_SENTINEL="${MOODLE_CODE_DIR}/.bootstrap.done"
 MOODLE_BOOTSTRAP_LOCK="${MOODLE_CODE_DIR}/.bootstrap.lock"
-MOODLE_BOOTSTRAP_STAGE="${MOODLE_CODE_DIR}/.bootstrap.stage"
 
 mkdir -p "${MOODLE_DATA_DIR}"
 chown -R "${MOODLE_RUNTIME_USER}:${MOODLE_RUNTIME_USER}" "${MOODLE_DATA_DIR}" || true  # nocheck: shell-or-true -- grandfathered: worked in practice; TODO: sharpen to catch only the exact tolerated error
@@ -51,6 +55,10 @@ moodle_bootstrap_code_dir() {
   rm -rf "${MOODLE_BOOTSTRAP_STAGE}"
   mkdir "${MOODLE_BOOTSTRAP_STAGE}"
   cp -a "${MOODLE_SOURCE_DIR}/." "${MOODLE_BOOTSTRAP_STAGE}/"
+  if moodle_code_dir_is_current; then
+    rm -rf "${MOODLE_BOOTSTRAP_STAGE}"
+    return 0
+  fi
   find "${MOODLE_CODE_DIR}" -mindepth 1 -maxdepth 1 ! -name "${MOODLE_BOOTSTRAP_LOCK##*/}" ! -name "${MOODLE_BOOTSTRAP_STAGE##*/}" -exec rm -rf {} +
   find "${MOODLE_BOOTSTRAP_STAGE}" -mindepth 1 -maxdepth 1 -exec mv -t "${MOODLE_CODE_DIR}" {} +
   rmdir "${MOODLE_BOOTSTRAP_STAGE}"

roles/web-app-moodle/tasks/03_ownership.yml

@@ -8,9 +8,12 @@
 # stream PHP files to FPM. Data dir stays restrictive (0770 / 0660)
 # because only Moodle FPM (running as MOODLE_RUNTIME_USER) needs to
 # read/write user-uploaded content.
+#
+# The walks prune MOODLE_BOOTSTRAP_STAGE: an entrypoint on the shared volume
+# tears that directory down while they run, so walking it hits vanished paths.
 
 - name: "✅ Ensure ownership is correct on Moodle code dir"
-  shell: "container exec --user root {{ MOODLE_CONTAINER_ADDRESS }} chown -R {{ MOODLE_RUNTIME_USER_GROUP }} {{ MOODLE_CODE_DIR }}"
+  shell: "container exec --user root {{ MOODLE_CONTAINER_ADDRESS }} find {{ MOODLE_CODE_DIR }} -path {{ MOODLE_BOOTSTRAP_STAGE }} -prune -o -exec chown -h {{ MOODLE_RUNTIME_USER_GROUP }} {} +"
   changed_when: false
 
 - name: "✅ Ensure ownership is correct on Moodle data dir"
@@ -18,11 +21,11 @@
   changed_when: false
 
 - name: "📁 Code dir: 0755 dirs (nginx-readable)"
-  shell: "container exec --user root {{ MOODLE_CONTAINER_ADDRESS }} find {{ MOODLE_CODE_DIR }} -type d -exec chmod 0755 {} \\;"
+  shell: "container exec --user root {{ MOODLE_CONTAINER_ADDRESS }} find {{ MOODLE_CODE_DIR }} -path {{ MOODLE_BOOTSTRAP_STAGE }} -prune -o -type d -exec chmod 0755 {} +"
   changed_when: false
 
 - name: "📁 Code dir: 0644 files (nginx-readable)"
-  shell: "container exec --user root {{ MOODLE_CONTAINER_ADDRESS }} find {{ MOODLE_CODE_DIR }} -type f -exec chmod 0644 {} \\;"
+  shell: "container exec --user root {{ MOODLE_CONTAINER_ADDRESS }} find {{ MOODLE_CODE_DIR }} -path {{ MOODLE_BOOTSTRAP_STAGE }} -prune -o -type f -exec chmod 0644 {} +"
   changed_when: false
 
 - name: "📁 Code dir: 0640 config.php (FPM-only)"

roles/web-app-moodle/templates/compose.yml.j2

@@ -14,6 +14,7 @@
         MOODLE_DATA_DIR: "{{ MOODLE_DATA_DIR }}"
         MOODLE_LOCAL_CACHE_DIR: "{{ MOODLE_LOCAL_CACHE_DIR }}"
         MOODLE_SOURCE_DIR: "{{ MOODLE_SOURCE_DIR }}"
+        MOODLE_BOOTSTRAP_STAGE: "{{ MOODLE_BOOTSTRAP_STAGE }}"
         MOODLE_RUNTIME_USER: "{{ MOODLE_RUNTIME_USER }}"
         MOODLE_OIDC_ENABLED: "{{ MOODLE_OIDC_ENABLED | bool | ternary('1','0') }}"
         MOODLE_OIDC_PLUGIN_RELPATH: "{{ MOODLE_OIDC_PLUGIN_RELPATH }}"

roles/web-app-moodle/vars/main.yml

@@ -8,6 +8,7 @@ MOODLE_DATA_DIR: "{{ lookup('volume', 'web-app-moodle', 'data').target }}"
 MOODLE_LOCAL_CACHE_DIR: "{{ lookup('volume', application_id, 'localcache').mounts[0].target }}"
 MOODLE_SOURCE_DIR: "/opt/moodle-source"
 MOODLE_ENTRYPOINT_PATH: "/usr/local/bin/moodle-entrypoint.sh"
+MOODLE_BOOTSTRAP_STAGE: "{{ [MOODLE_CODE_DIR, '.bootstrap.stage'] | path_join }}"
 
 MOODLE_BUILD_DIR: "build"
 MOODLE_BUILD_FILES:

tests/unit/python/roles/web-app-moodle/files/build/test_moodle_entrypoint.py

@@ -13,6 +13,12 @@ ENTRYPOINT = PROJECT_ROOT / "roles/web-app-moodle/files/build/moodle-entrypoint.
 
 FPM_STUB = "#!/usr/bin/env bash\nexit 0\n"
 
+CP_STUB = (
+    "#!/usr/bin/env bash\n"
+    'printf %s "$SIBLING_RELEASE" > "${MOODLE_CODE_DIR}/.bootstrap.done"\n'
+    'exec @REAL_CP@ "$@"\n'
+)
+
 LAYOUT_4 = {"version.php": "4.5.11", "auth/oidc/version.php": "4.5.8"}
 LAYOUT_5 = {"public/version.php": "5.3.0", "public/auth/oidc/version.php": "5.2.1"}
 
@@ -24,22 +30,36 @@ class TestMoodleEntrypoint(unittest.TestCase):
         self.root = Path(tmp.name)
         self.source = self.root / "source"
         self.code = self.root / "code"
-        bin_dir = self.root / "bin"
-        bin_dir.mkdir()
-        stub = bin_dir / "php-fpm"
+        self.stage = self.code / ".bootstrap.stage"
+        self.bin_dir = self.root / "bin"
+        self.bin_dir.mkdir()
+        self.real_cp = shutil.which("cp")
+        stub = self.bin_dir / "php-fpm"
         stub.write_text(FPM_STUB)
         stub.chmod(0o755)
         self.env = dict(os.environ)
         self.env.update(
-            PATH=f"{bin_dir}{os.pathsep}{self.env['PATH']}",
+            PATH=f"{self.bin_dir}{os.pathsep}{self.env['PATH']}",
             MOODLE_CODE_DIR=str(self.code),
             MOODLE_DATA_DIR=str(self.root / "data"),
             MOODLE_LOCAL_CACHE_DIR=str(self.root / "localcache"),
             MOODLE_SOURCE_DIR=str(self.source),
+            MOODLE_BOOTSTRAP_STAGE=str(self.stage),
             MOODLE_RUNTIME_USER=pwd.getpwuid(os.getuid()).pw_name,
             MOODLE_VERSION_FILE="public/version.php",
         )
 
+    def sibling_finishes_during_the_copy(self, release):
+        """Stub ``cp`` so another container marks the volume done mid-bootstrap.
+
+        Args:
+            release: release the sibling records in the bootstrap sentinel.
+        """
+        stub = self.bin_dir / "cp"
+        stub.write_text(CP_STUB.replace("@REAL_CP@", self.real_cp))
+        stub.chmod(0o755)
+        self.env["SIBLING_RELEASE"] = release
+
     def write(self, tree, files):
         for name, content in files.items():
             path = tree / name
@@ -109,6 +129,16 @@ class TestMoodleEntrypoint(unittest.TestCase):
         self.assertEqual(self.shipped_files_in_the_volume(), set(LAYOUT_4))
         self.assertEqual(read_text(str(self.code / ".bootstrap.done")), "4.5.11")
 
+    def test_a_sibling_that_finishes_first_keeps_its_tree(self):
+        self.ship(LAYOUT_5)
+        installed = {"public/local/installed/version.php": "written by the deploy"}
+        self.write(self.code, installed)
+        self.sibling_finishes_during_the_copy("5.3.0")
+        self.start("5.3.0")
+        self.assertTrue((self.code / "public/local/installed/version.php").is_file())
+        self.assertFalse(self.stage.exists())
+        self.assertEqual(read_text(str(self.code / ".bootstrap.done")), "5.3.0")
+
     def test_an_image_older_than_the_recorded_release_refuses_to_replace_it(self):
         self.ship(LAYOUT_5)
         self.start("5.10.0")