mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-10 04:36:40 +00:00
kevinveenbirkenbachandClaude Opus 5
40688e4ded
`request()` ended in `touch(exist_ok=True)`, and on an existing file that
refreshes the mtime. `next_queued()` sorts by mtime and serves the oldest
marker, so re-requesting a build that is already queued moved it behind
everything else. The server calls `request()` on every visit, and a visitor
waiting for a page polls every thirty seconds, so waiting for a build was
what prevented it:
queued de, then a tag : ['deployed:de', 'v14.2.0'] -> builds deployed:de
after one poll on de : ['v14.2.0', 'deployed:de'] -> builds v14.2.0
One poll is enough. In the docs suite the spec before it deliberately clicks
an unbuilt release tag, so another hour-long build is always sitting there to
be overtaken by.
Observed in run 36596159856: `/deployed/de/` answered 202 and never 200 for
two hours and four minutes in one deployment and two hours fifty-seven in
another, from those jobs' own rescue artifacts, while the suite budget is
fifty-nine minutes on clearnet and eighty-nine on tor.
This removes the starvation, not the arithmetic: a full Sphinx build costs
sixty to ninety minutes here and the suite needs two of them in sequence,
English then German. Whether one round now fits is for a run to say.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
114 lines
3.8 KiB
Python
114 lines
3.8 KiB
Python
"""The build queue, the cross-replica builder lock and the daemon loop.
|
|
|
|
``Queue`` is a mixin of :class:`infinito_docs.library.Library`. A marker is a
|
|
file named ``version`` or ``version:code``; the background lane holds the
|
|
languages nobody is waiting on.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from infinito_docs.builder import DEPLOYED, QUEUE_SEPARATOR
|
|
|
|
POLL_SECONDS = 2
|
|
BACKGROUND_LANE = "background"
|
|
|
|
|
|
class Queue:
|
|
def request(self, version, code=None, background=False):
|
|
"""Queue a build of ``version``, or of one of its translated sites.
|
|
|
|
Args:
|
|
version: ``latest`` or a release tag.
|
|
code: ISO 639-1 code to build instead of the version's own site.
|
|
background: queue behind every language a visitor asked for.
|
|
"""
|
|
head, _ = self.refs()
|
|
if not head and version != DEPLOYED:
|
|
return
|
|
if code is None:
|
|
if self._current(version, head):
|
|
return
|
|
marker = version
|
|
elif not self.translates(version, code) or self._translation_current(
|
|
version, code, head
|
|
):
|
|
return
|
|
else:
|
|
marker = f"{version}{QUEUE_SEPARATOR}{code}"
|
|
lane = self.queue / BACKGROUND_LANE if background else self.queue
|
|
lane.mkdir(parents=True, exist_ok=True)
|
|
if background and (self.queue / marker).exists():
|
|
return
|
|
queued = lane / marker
|
|
if queued.exists():
|
|
return
|
|
queued.touch()
|
|
|
|
def _dequeue(self, marker):
|
|
"""Drop ``marker`` from both lanes.
|
|
|
|
Args:
|
|
marker: queue file name, ``version`` or ``version:code``.
|
|
"""
|
|
(self.queue / marker).unlink(missing_ok=True)
|
|
(self.queue / BACKGROUND_LANE / marker).unlink(missing_ok=True)
|
|
|
|
def next_queued(self):
|
|
"""Return the oldest marker a visitor is waiting on, else the oldest background one."""
|
|
for lane in (self.queue, self.queue / BACKGROUND_LANE):
|
|
if not lane.is_dir():
|
|
continue
|
|
waiting = sorted(
|
|
(marker for marker in lane.iterdir() if marker.is_file()),
|
|
key=lambda marker: marker.stat().st_mtime,
|
|
)
|
|
if waiting:
|
|
return waiting[0].name
|
|
return None
|
|
|
|
def acquire_builder(self):
|
|
"""Try to become the builder of all replicas without blocking.
|
|
|
|
Returns:
|
|
``True`` while this process holds the builder lock.
|
|
"""
|
|
if self._lock_handle is not None:
|
|
return True
|
|
self.lock_file.parent.mkdir(parents=True, exist_ok=True)
|
|
handle = self.lock_file.open("a+")
|
|
try:
|
|
fcntl.lockf(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
handle.close()
|
|
return False
|
|
self._lock_handle = handle
|
|
return True
|
|
|
|
def run_builder(self, interval):
|
|
while not self.acquire_builder():
|
|
time.sleep(POLL_SECONDS)
|
|
if self.snapshot.is_dir():
|
|
self.request(DEPLOYED)
|
|
next_fetch = 0.0
|
|
while True:
|
|
if time.monotonic() >= next_fetch:
|
|
try:
|
|
self.fetch()
|
|
except (OSError, subprocess.CalledProcessError) as exc:
|
|
print(f"fetch of {self.repository} failed: {exc}", file=sys.stderr)
|
|
next_fetch = time.monotonic() + interval
|
|
marker = self.next_queued()
|
|
if marker is None:
|
|
time.sleep(POLL_SECONDS)
|
|
continue
|
|
version, separator, code = marker.partition(QUEUE_SEPARATOR)
|
|
if separator:
|
|
self.build_language(version, code)
|
|
else:
|
|
self.build(version)
|