mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-06 10:59:59 +00:00
kevinveenbirkenbachandClaude Opus 5.5
4cba7e7c2f
An http_regex source read one document, so a tag whose parts upstream keeps in two files could only carry the second part as a literal. Every pin also fetched its document on its own: a failed request left one pin of a release behind while the others moved. url now takes a list whose bodies are searched as one text, and every document is fetched once per run, so all pins that read it resolve against the same snapshot. The HTTP reads move into utils/update/fetch.py. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
47 lines
1.3 KiB
Python
47 lines
1.3 KiB
Python
"""Read the documents a declared version source names.
|
|
|
|
Every document is fetched once per run, so the pins that read the same release
|
|
manifest resolve against one snapshot of it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import urllib.request
|
|
|
|
TIMEOUT_SECONDS = 30
|
|
USER_AGENT = "infinito-nexus-version-source"
|
|
|
|
|
|
def get(url: str, headers: dict[str, str] | None = None) -> bytes:
|
|
"""Return the body of a declared version source.
|
|
|
|
Args:
|
|
url: https URL to read.
|
|
headers: request headers beside the user agent.
|
|
"""
|
|
request = urllib.request.Request( # noqa: S310 - https URL of a declared version source
|
|
url, headers={"User-Agent": USER_AGENT, **(headers or {})}
|
|
)
|
|
with urllib.request.urlopen( # noqa: S310 - https URL of a declared version source
|
|
request, timeout=TIMEOUT_SECONDS
|
|
) as response:
|
|
return response.read()
|
|
|
|
|
|
@functools.cache
|
|
def document(url: str) -> str:
|
|
"""Return the text of *url*, fetched once per run."""
|
|
return get(url).decode("utf-8", "replace")
|
|
|
|
|
|
def documents(urls: str | list[str]) -> str:
|
|
"""Return the text of one URL, or of several as one text.
|
|
|
|
Args:
|
|
urls: a URL, or several whose bodies are searched together.
|
|
"""
|
|
return "\n".join(
|
|
document(url) for url in ([urls] if isinstance(urls, str) else urls)
|
|
)
|