mirror of
https://github.com/kevinveenbirkenbach/computer-playbook.git
synced 2026-11-08 19:56:53 +00:00
funkwhale has two S3 surfaces sharing one private bucket. Audio goes out as X-Accel-Redirect and the front nginx fetches it server-side, but attachments (instance logo, avatars, covers) land in the API JSON as absolute URLs the BROWSER dials. AWS_S3_ENDPOINT_URL drives the boto3 client, so with the internal endpoint django-storages signed URLs for http://seaweedfs:8333, a host no browser can reach. Point the client at the new bucket-less public_endpoint key. Measured against the live gateway: a URL signed for seaweedfs:8333 and sent to the public host answers 403 SignatureDoesNotMatch, while one signed for the public host reaches bucket lookup — query-string SigV4 binds the Host, so the signing host and the fetch host must be the same string. AWS_S3_CUSTOM_DOMAIN cannot serve this: storages/backends/s3.py:689 returns UNSIGNED without a CloudFront signer, and the bucket also holds the music library that music/views.py exists to gate. AWS_CA_BUNDLE follows, because the public endpoint is TLS on the local CA and botocore reads its trust store only from that variable (configprovider.py:59) — sys-svc-compose-ca injects SSL_CERT_FILE, REQUESTS_CA_BUNDLE, CURL_CA_BUNDLE and NODE_EXTRA_CA_CERTS, never this one. It takes the bundle rather than the bare root cert, since AWS_CA_BUNDLE replaces the default store. AWS_QUERYSTRING_AUTH=True fixes a pre-existing break: upstream defaults it to `not PROXY_MEDIA`, which produced unsigned URLs against a private bucket. The new front nginx.conf.j2 serves /_protected/media, which the stock template resolves with `alias ${MEDIA_ROOT}/$1;` and therefore 404s for every S3-backed listen. web-svc-seaweedfs keeps no onion vhost. Its tor-contract reason now states the mechanical cause instead of the old claim that the role is never a Tor-browser surface, which decidim and funkwhale have made false: the node hidden service publishes http only while sys-front-tls keys listen/ssl on application_id, so an onion vhost would listen on 443 against a port-80 hidden service. Attachment fetches therefore still fail on an onion pick, and the spec is left to fail there rather than record the miss and pass. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
301 lines
11 KiB
Python
301 lines
11 KiB
Python
"""Unit tests for ``plugins/lookup/objstore.py``.
|
|
|
|
The lookup branches on which object-store engine (seaweedfs|minio) the
|
|
consumer has enabled and yields a uniform S3 connection payload.
|
|
``get_merged_applications`` is mocked so the tests stay hermetic.
|
|
"""
|
|
|
|
import importlib.util
|
|
import unittest
|
|
import unittest.mock as mock
|
|
from unittest.mock import patch
|
|
|
|
from ansible.errors import AnsibleError
|
|
|
|
from . import PROJECT_ROOT
|
|
|
|
|
|
def _load_module(rel_path: str, name: str):
|
|
path = PROJECT_ROOT / rel_path
|
|
spec = importlib.util.spec_from_file_location(name, str(path))
|
|
mod = importlib.util.module_from_spec(spec)
|
|
assert spec and spec.loader
|
|
spec.loader.exec_module(mod)
|
|
return mod
|
|
|
|
|
|
class _DummyTemplar:
|
|
def __init__(self, available_variables):
|
|
self.available_variables = available_variables
|
|
|
|
|
|
class ObjstoreLookupTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.mod = _load_module("plugins/lookup/objstore.py", "lookup_objstore")
|
|
|
|
def _make_lookup(self, available_vars: dict):
|
|
lm = self.mod.LookupModule()
|
|
lm._templar = _DummyTemplar(available_vars)
|
|
lm._loader = mock.MagicMock()
|
|
return lm
|
|
|
|
@staticmethod
|
|
def _fake_entity_name(role_name: str) -> str:
|
|
"""
|
|
Make entity resolution deterministic for unit tests (no filesystem access).
|
|
Mirrors the typical behavior for your role naming.
|
|
"""
|
|
role_name = role_name.strip()
|
|
for prefix in ("web-app-", "web-svc-", "svc-db-", "svc-", "persona-"):
|
|
if role_name.startswith(prefix):
|
|
return role_name[len(prefix) :]
|
|
return role_name
|
|
|
|
def _run(self, terms, applications, vars_=None):
|
|
if vars_ is None:
|
|
vars_ = {"DIR_COMPOSITIONS": "/opt/compose/"}
|
|
lookup = self._make_lookup(vars_)
|
|
with (
|
|
mock.patch.object(self.mod, "lookup_loader") as loader_mock,
|
|
patch.object(self.mod, "entity_name", side_effect=self._fake_entity_name),
|
|
):
|
|
loader_mock.get.return_value = mock.MagicMock(
|
|
run=lambda *_a, **_k: [applications]
|
|
)
|
|
return lookup.run(terms, variables=vars_)
|
|
|
|
def test_invalid_terms_raises(self):
|
|
with self.assertRaises(AnsibleError):
|
|
self._run([], {})
|
|
|
|
with self.assertRaises(AnsibleError):
|
|
self._run(["a", "b", "c"], {})
|
|
|
|
def test_empty_consumer_id_raises(self):
|
|
with self.assertRaises(AnsibleError):
|
|
self._run([" "], {})
|
|
|
|
def test_missing_dir_compositions_raises(self):
|
|
with self.assertRaises(AnsibleError):
|
|
self._run(["web-app-foo"], {"web-app-foo": {"services": {}}}, vars_={})
|
|
|
|
def test_no_engine_enabled_returns_disabled_payload(self):
|
|
applications = {"web-app-foo": {"services": {}}}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(out["id"], "")
|
|
self.assertEqual(out["engine"], "")
|
|
self.assertFalse(out["enabled"])
|
|
self.assertFalse(out["shared"])
|
|
self.assertFalse(out["local"])
|
|
self.assertEqual(out["host"], "")
|
|
self.assertEqual(out["port"], "")
|
|
self.assertEqual(out["network"], "")
|
|
self.assertEqual(out["container"], "")
|
|
self.assertEqual(out["bucket"], "foo")
|
|
self.assertEqual(out["access_key"], "foo")
|
|
self.assertEqual(out["secret_key"], "")
|
|
self.assertEqual(out["region"], "us-east-1")
|
|
self.assertEqual(out["env"], "")
|
|
self.assertEqual(out["volume"], "")
|
|
self.assertEqual(out["endpoint"], "")
|
|
self.assertEqual(out["url"], "")
|
|
self.assertEqual(out["public_domain"], "")
|
|
self.assertEqual(out["public_bucket"], "")
|
|
self.assertEqual(out["public_endpoint"], "")
|
|
self.assertEqual(out["public_url"], "")
|
|
self.assertEqual(out["reach_host"], "127.0.0.1")
|
|
|
|
self.assertEqual(self._run(["web-app-foo", "url"], applications)[0], "")
|
|
|
|
def test_seaweedfs_shared_uses_central_provider_values(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {"seaweedfs": {"enabled": True, "shared": True}},
|
|
"secrets": {"credentials": {"objstore_secret_key": "sk"}},
|
|
},
|
|
"web-svc-seaweedfs": {
|
|
"services": {
|
|
"seaweedfs": {"name": "seaweedfs-central", "api_port": 8334}
|
|
},
|
|
"domains": {"canonical": {"api": "api.seaweedfs.s3.example.com"}},
|
|
},
|
|
}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(out["id"], "web-svc-seaweedfs")
|
|
self.assertEqual(out["engine"], "seaweedfs")
|
|
self.assertTrue(out["enabled"])
|
|
self.assertTrue(out["shared"])
|
|
self.assertFalse(out["local"])
|
|
self.assertEqual(out["host"], "seaweedfs-central")
|
|
self.assertEqual(out["port"], 8334)
|
|
self.assertEqual(out["network"], "seaweedfs")
|
|
self.assertEqual(out["container"], "seaweedfs-central")
|
|
self.assertEqual(out["volume"], "")
|
|
self.assertEqual(out["endpoint"], "seaweedfs-central:8334")
|
|
self.assertEqual(out["url"], "http://seaweedfs-central:8334")
|
|
self.assertEqual(out["public_domain"], "api.seaweedfs.s3.example.com")
|
|
self.assertEqual(out["public_bucket"], "api.seaweedfs.s3.example.com/foo")
|
|
self.assertEqual(out["public_endpoint"], "https://api.seaweedfs.s3.example.com")
|
|
self.assertEqual(out["public_url"], "https://api.seaweedfs.s3.example.com/foo")
|
|
|
|
self.assertEqual(
|
|
self._run(["web-app-foo", "url"], applications)[0],
|
|
"http://seaweedfs-central:8334",
|
|
)
|
|
|
|
def test_named_canonical_siblings_do_not_win_over_the_api(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {"minio": {"enabled": True, "shared": True}},
|
|
"secrets": {"credentials": {"objstore_secret_key": "sk"}},
|
|
},
|
|
"web-app-minio": {
|
|
"services": {"minio": {"name": "minio-central"}},
|
|
"domains": {
|
|
"canonical": {
|
|
"console": "console.minio.s3.example.com",
|
|
"api": "api.minio.s3.example.com",
|
|
}
|
|
},
|
|
},
|
|
}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(out["public_domain"], "api.minio.s3.example.com")
|
|
self.assertEqual(out["public_endpoint"], "https://api.minio.s3.example.com")
|
|
|
|
def test_credentialed_url_percent_encodes_both_halves(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {"seaweedfs": {"enabled": True, "shared": True}},
|
|
"secrets": {"credentials": {"objstore_secret_key": "a/b+c=d"}},
|
|
},
|
|
"web-svc-seaweedfs": {
|
|
"services": {
|
|
"seaweedfs": {"name": "seaweedfs-central", "api_port": 8334}
|
|
},
|
|
},
|
|
}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(
|
|
out["credentialed_url"],
|
|
"http://foo:a%2Fb%2Bc%3Dd@seaweedfs-central:8334",
|
|
)
|
|
self.assertEqual(out["url"], "http://seaweedfs-central:8334")
|
|
|
|
def test_credentialed_url_is_empty_without_an_engine(self):
|
|
applications = {"web-app-foo": {"services": {}}}
|
|
self.assertEqual(
|
|
self._run(["web-app-foo", "credentialed_url"], applications)[0], ""
|
|
)
|
|
|
|
def test_public_url_scheme_follows_provider_tls(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {"minio": {"enabled": True, "shared": True}},
|
|
"secrets": {"credentials": {"objstore_secret_key": "sk"}},
|
|
},
|
|
"web-app-minio": {
|
|
"services": {"minio": {"name": "minio-central"}},
|
|
"server": {"tls": {"enabled": False}},
|
|
"domains": {"canonical": {"api": "api.minio.s3.example.com"}},
|
|
},
|
|
}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(out["public_domain"], "api.minio.s3.example.com")
|
|
self.assertEqual(out["public_url"], "http://api.minio.s3.example.com/foo")
|
|
|
|
def test_seaweedfs_dedicated_matches_embedded_instance(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {"seaweedfs": {"enabled": True, "shared": False}},
|
|
"secrets": {"credentials": {"objstore_secret_key": "sk"}},
|
|
}
|
|
}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(out["id"], "web-svc-seaweedfs")
|
|
self.assertEqual(out["engine"], "seaweedfs")
|
|
self.assertTrue(out["enabled"])
|
|
self.assertFalse(out["shared"])
|
|
self.assertTrue(out["local"])
|
|
self.assertEqual(out["host"], "seaweedfs")
|
|
self.assertEqual(out["port"], 8333)
|
|
self.assertEqual(out["network"], "foo")
|
|
self.assertEqual(out["container"], "foo-seaweedfs")
|
|
self.assertEqual(out["volume"], "foo_seaweedfs")
|
|
self.assertEqual(out["url"], "http://seaweedfs:8333")
|
|
self.assertEqual(out["env"], "/opt/compose/foo/.env/objstore.env")
|
|
self.assertEqual(out["image"], "chrislusf/seaweedfs")
|
|
self.assertEqual(out["version"], "latest")
|
|
|
|
def test_minio_shared_defaults_port_9000(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {"minio": {"enabled": True, "shared": True}},
|
|
"secrets": {"credentials": {"objstore_secret_key": "sk"}},
|
|
},
|
|
"web-app-minio": {"services": {"minio": {"name": "minio-central"}}},
|
|
}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(out["id"], "web-app-minio")
|
|
self.assertEqual(out["engine"], "minio")
|
|
self.assertTrue(out["enabled"])
|
|
self.assertTrue(out["shared"])
|
|
self.assertFalse(out["local"])
|
|
self.assertEqual(out["host"], "minio-central")
|
|
self.assertEqual(out["port"], 9000)
|
|
self.assertEqual(out["network"], "minio")
|
|
self.assertEqual(out["url"], "http://minio-central:9000")
|
|
|
|
def test_multiple_engines_enabled_raises(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {
|
|
"seaweedfs": {"enabled": True, "shared": False},
|
|
"minio": {"enabled": True, "shared": False},
|
|
}
|
|
}
|
|
}
|
|
|
|
with self.assertRaises(AnsibleError):
|
|
self._run(["web-app-foo"], applications)
|
|
|
|
def test_secret_key_and_public_passthrough(self):
|
|
applications = {
|
|
"web-app-foo": {
|
|
"services": {
|
|
"seaweedfs": {"enabled": True, "shared": False, "public": True}
|
|
},
|
|
"secrets": {"credentials": {"objstore_secret_key": "s3cr3t"}},
|
|
}
|
|
}
|
|
|
|
out = self._run(["web-app-foo"], applications)[0]
|
|
|
|
self.assertEqual(out["secret_key"], "s3cr3t")
|
|
self.assertTrue(out["public"])
|
|
self.assertEqual(out["access_key"], "foo")
|
|
self.assertEqual(out["bucket"], "foo")
|
|
|
|
self.assertEqual(
|
|
self._run(["web-app-foo", "secret_key"], applications)[0], "s3cr3t"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|