From 0b7a8a95dc45caa3d7e73a19b849277c16dcffb4 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Sat, 19 Sep 2026 11:50:29 +0200 Subject: [PATCH] feat(lmstudio): seed the model store once, before the daemons index it c6e8990e25 kept the daemon's auth key off the shared volume and made the red job green, but it left multi-replica operation as broken as it found it: three daemons still shared one /root/.lmstudio, where LM Studio rewrites extensions/plugins with rm -rf at every start, writes settings.json in place and resets it to defaults on a torn read, and re-copies a 102 MB bin/lms while peers execute it. Splitting the tree by access pattern fixes that. /root/.lmstudio is node-local, because it is per-daemon state the daemon rewrites in place. models/ stays shared, because a blob is written once and read forever. That leaves one question, and it is temporal rather than mechanical: the daemon discovers models by an unconditional scan at startup and by an inotify watcher, and inotify never sees writes made by another NFS client. So a model seeded after a daemon started is invisible to it, whichever tool wrote the file. The previous design answered that with `container service update --force`, which restarts the whole inference backend svc-ai-litellm points at, on every deploy. The seed now runs before the stack does. 00_core.yml places the blob on the NFS export ahead of the compose include that deploys the stack, so every daemon finds a complete tree at its own startup scan and nothing has to be restarted. Measured on a three-node swarm with NFS: Fetch the models into the shared store 136.14s one 531 MB download Pre-pull on mgr-01 0.69s Pre-pull on wrk-01 0.50s Pre-pull on wrk-02 0.49s Half a second is not a download. All three daemons recognised the pre-placed model, so the cluster now stores and transfers it once instead of three times. The nested mount across two devices raises no EXDEV, and the per-node disk guard correctly skipped itself four times because the pre-seed covers that check. Vendor coupling is gone from the role. `repo` and `url` are declared fields rather than pieces cut out of a HuggingFace URL with a regex; a model on any host needs only those two. `lms get` still knows only HuggingFace, so `source` stays for it, and a non-HF model simply leaves it unset: the blob is seeded and the startup scan picks it up. The external check follows, comparing against the HuggingFace API only for entries whose url actually points there. The guard arithmetic was wrong and is now derived rather than guessed. Peak usage is the missing bytes plus the largest single model, not twice the declared total, so the read-back moved ahead of the check and subtracts what the store already holds. Before, the requirement grew after every successful seed. Staging moved beside the tree instead of inside it, so an aborted transfer cannot leave orphans where the indexer scans. A deployment test ships with it and states its findings rather than its exit code: it fails on `authPacket` by naming the shared key, on a missing or mismatching blob by naming path and digest, and on a model that is on disk but absent from /v1/models by saying the startup scan did not happen. Gone with the restart: import.sh and its container-cp suppression, seed.yml, rescan.yml, and the disk-copy factor of three that only existed because the import staged through the container. Residual: compose is untested here. There the pre-seed is off, the node-local guard is the one that runs, and `lms get` is the only fetch, so a non-HF model is not loadable in that mode. Seeding node-local volumes is the follow-up that closes it. Co-Authored-By: Claude Opus 5 (1M context) --- group_vars/all/16_ai.yml | 2 + roles/svc-ai-lmstudio/files/shell/import.sh | 34 ----- roles/svc-ai-lmstudio/files/test/.nocheck | 1 + roles/svc-ai-lmstudio/files/test/test.sh | 100 +++++++++++++++ roles/svc-ai-lmstudio/meta/tests.yml | 4 + roles/svc-ai-lmstudio/meta/volumes.yml | 14 +-- roles/svc-ai-lmstudio/tasks/00_core.yml | 8 ++ .../svc-ai-lmstudio/tasks/01_manager_ops.yml | 32 ++--- roles/svc-ai-lmstudio/tasks/utils/cid.yml | 22 ++++ roles/svc-ai-lmstudio/tasks/utils/nodes.yml | 29 +++++ .../tasks/utils/preload/node.yml | 42 +++++++ .../utils/{preload.yml => preload/pull.yml} | 0 .../tasks/utils/{ => preload}/recover.yml | 2 +- roles/svc-ai-lmstudio/tasks/utils/preseed.yml | 118 ++++++++++++++++++ roles/svc-ai-lmstudio/tasks/utils/seed.yml | 65 ---------- roles/svc-ai-lmstudio/templates/test.env.j2 | 5 + roles/svc-ai-lmstudio/vars/main.yml | 20 +-- .../test_ai_local_models_downloadable.py | 40 +++--- 18 files changed, 380 insertions(+), 158 deletions(-) delete mode 100644 roles/svc-ai-lmstudio/files/shell/import.sh create mode 100644 roles/svc-ai-lmstudio/files/test/.nocheck create mode 100644 roles/svc-ai-lmstudio/files/test/test.sh create mode 100644 roles/svc-ai-lmstudio/meta/tests.yml create mode 100644 roles/svc-ai-lmstudio/tasks/utils/cid.yml create mode 100644 roles/svc-ai-lmstudio/tasks/utils/nodes.yml create mode 100644 roles/svc-ai-lmstudio/tasks/utils/preload/node.yml rename roles/svc-ai-lmstudio/tasks/utils/{preload.yml => preload/pull.yml} (100%) rename roles/svc-ai-lmstudio/tasks/utils/{ => preload}/recover.yml (95%) create mode 100644 roles/svc-ai-lmstudio/tasks/utils/preseed.yml delete mode 100644 roles/svc-ai-lmstudio/tasks/utils/seed.yml create mode 100644 roles/svc-ai-lmstudio/templates/test.env.j2 diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 8c47b2dcb1..c79faa45c5 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -24,6 +24,8 @@ LITELLM_LMSTUDIO_BACKEND: >- AI_LOCAL_MODELS: - alias: qwen2.5:0.5b name: qwen2.5-0.5b-instruct + repo: lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF + url: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-Q8_0.gguf source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF file: Qwen2.5-0.5B-Instruct-Q8_0.gguf sha256: "1d96e30253a76970864323987e3e77eeaf170312952fd61d18d343c25cfd6e52" diff --git a/roles/svc-ai-lmstudio/files/shell/import.sh b/roles/svc-ai-lmstudio/files/shell/import.sh deleted file mode 100644 index 5e9e848d40..0000000000 --- a/roles/svc-ai-lmstudio/files/shell/import.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Hand a pre-fetched GGUF to LM Studio, idempotent. -# Runs ON THE HOSTING NODE (delegated); targets the local container via -# `container exec` / `container cp`. -# -# Required env, supplied by the calling Ansible task: -# LMSTUDIO_LOCAL_CID resolved container id, local to this node -# MODEL_FILE node-side path to the fetched .gguf -# MODEL_USER_REPO HuggingFace "user/repo" the file belongs to -# MODELS_DIR models folder inside the container, under the -# volume target declared in meta/volumes.yml -set -euo pipefail - -ADDRESS="${LMSTUDIO_LOCAL_CID:?LMSTUDIO_LOCAL_CID env var (container id) required}" -BASE="$(basename "${MODEL_FILE}")" -TARGET="${MODELS_DIR}/${MODEL_USER_REPO}/${BASE}" - -if container exec "$ADDRESS" test -f "${TARGET}"; then - echo "PRESENT:${BASE}" - exit 0 -fi - -if [ ! -f "${MODEL_FILE}" ]; then - echo "ABSENT:${BASE}" - exit 0 -fi - -# nocheck: container-cp - MODEL_FILE is fetched on the node this script runs on -container cp "${MODEL_FILE}" "${ADDRESS}:/tmp/${BASE}" - -printf 'y\n' | container exec -i "$ADDRESS" lms import "/tmp/${BASE}" --yes --copy --user-repo "${MODEL_USER_REPO}" -container exec "$ADDRESS" rm -f "/tmp/${BASE}" -echo "IMPORTED:${BASE}" diff --git a/roles/svc-ai-lmstudio/files/test/.nocheck b/roles/svc-ai-lmstudio/files/test/.nocheck new file mode 100644 index 0000000000..a0fdb0bf8c --- /dev/null +++ b/roles/svc-ai-lmstudio/files/test/.nocheck @@ -0,0 +1 @@ +# nocheck: role-files-layout - Role test harness with test.sh as its entry point; the CLI runner stages roles//files/test/ by path. diff --git a/roles/svc-ai-lmstudio/files/test/test.sh b/roles/svc-ai-lmstudio/files/test/test.sh new file mode 100644 index 0000000000..fc83bfc748 --- /dev/null +++ b/roles/svc-ai-lmstudio/files/test/test.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# +# Env (rendered into test.env from templates/test.env.j2): +# LMSTUDIO_CONTAINER resolved container id on this node +# LMSTUDIO_PORT the daemon's HTTP port +# LMSTUDIO_EXPECTED_MODELS JSON array of model names the daemon must serve +# LMSTUDIO_EXPECTED_BLOBS JSON array of / paths under models/ +# LMSTUDIO_EXPECTED_DIGESTS JSON array of sha256, aligned with the blobs +# RETRIES attempts (default 30) +# SLEEP_SECONDS wait between (default 10) + +set -uo pipefail + +RETRIES="${RETRIES:-30}" +SLEEP_SECONDS="${SLEEP_SECONDS:-10}" +MODELS_DIR=/root/.lmstudio/models + +[ -n "${LMSTUDIO_CONTAINER}" ] || { + echo "[FATAL] LMSTUDIO_CONTAINER unset; the harness did not resolve a container" >&2 + exit 2 +} + +in_container() { + container exec "${LMSTUDIO_CONTAINER}" "$@" +} + +json_items() { + python3 -c 'import json,sys; [print(x) for x in json.loads(sys.argv[1])]' "$1" +} + +serves_model() { + in_container curl -fsS --connect-timeout 5 --max-time 30 "http://127.0.0.1:${LMSTUDIO_PORT}/v1/models" 2>/dev/null | + python3 -c 'import json,sys; d=json.load(sys.stdin); print("\n".join(m.get("id","") for m in d.get("data",[])))' 2>/dev/null | + grep -qxF "$1" +} + +failed=0 + +echo "=== 1. the CLI authenticates against its own daemon ===" +attempt=1 +auth_out="" +while [ "${attempt}" -le "${RETRIES}" ]; do + if auth_out="$(in_container /app/.bundle/lms ls 2>&1)"; then + echo "[OK] lms ls authenticated" + break + fi + if printf '%s' "${auth_out}" | grep -q 'authPacket'; then + echo "[FAIL] the daemon rejected its own CLI: $(printf '%s' "${auth_out}" | grep -m1 authPacket)" >&2 + echo " a second daemon shares /root/.lmstudio and rewrote .internal/lms-key-2" >&2 + exit 1 + fi + attempt=$((attempt + 1)) + [ "${attempt}" -le "${RETRIES}" ] && sleep "${SLEEP_SECONDS}" +done +if [ "${attempt}" -gt "${RETRIES}" ]; then + echo "[FAIL] lms ls never succeeded after ${RETRIES} attempts: ${auth_out}" >&2 + exit 1 +fi + +echo "=== 2. every declared blob is in place with its declared digest ===" +mapfile -t blobs < <(json_items "${LMSTUDIO_EXPECTED_BLOBS}") +mapfile -t digests < <(json_items "${LMSTUDIO_EXPECTED_DIGESTS}") +for i in "${!blobs[@]}"; do + path="${MODELS_DIR}/${blobs[$i]}" + if ! in_container test -f "${path}"; then + echo "[FAIL] ${blobs[$i]} is not in the model store" >&2 + failed=1 + continue + fi + actual="$(in_container sha256sum "${path}" 2>/dev/null | awk '{print $1}')" + if [ "${actual}" != "${digests[$i]}" ]; then + echo "[FAIL] ${blobs[$i]} hashes to ${actual}, declared ${digests[$i]}" >&2 + failed=1 + continue + fi + echo "[OK] ${blobs[$i]} present and matching" +done + +echo "=== 3. the daemon serves every declared model ===" +mapfile -t models < <(json_items "${LMSTUDIO_EXPECTED_MODELS}") +for model in "${models[@]}"; do + attempt=1 + while [ "${attempt}" -le "${RETRIES}" ]; do + if serves_model "${model}"; then + echo "[OK] ${model} is served" + break + fi + attempt=$((attempt + 1)) + [ "${attempt}" -le "${RETRIES}" ] && sleep "${SLEEP_SECONDS}" + done + if [ "${attempt}" -gt "${RETRIES}" ]; then + echo "[FAIL] ${model} is on disk but /v1/models never listed it" >&2 + echo " the daemon did not index the store it was given at startup" >&2 + failed=1 + fi +done + +[ "${failed}" -eq 0 ] || exit 1 +echo "[OK] svc-ai-lmstudio serves every declared model from the seeded store" diff --git a/roles/svc-ai-lmstudio/meta/tests.yml b/roles/svc-ai-lmstudio/meta/tests.yml new file mode 100644 index 0000000000..712dbdf2f8 --- /dev/null +++ b/roles/svc-ai-lmstudio/meta/tests.yml @@ -0,0 +1,4 @@ +--- +cli: + container_service_key: lmstudio + timeout: 900 diff --git a/roles/svc-ai-lmstudio/meta/volumes.yml b/roles/svc-ai-lmstudio/meta/volumes.yml index ae7259ecdf..0250216548 100644 --- a/roles/svc-ai-lmstudio/meta/volumes.yml +++ b/roles/svc-ai-lmstudio/meta/volumes.yml @@ -1,17 +1,17 @@ --- -# nocheck: embedded-state-on-nfs model blobs written once on pull and read back afterwards lmstudio_data: type: volume + name: lmstudio_data + # nocheck: nfs-false-data-loss Reason: per-daemon state LM Studio rewrites in place (lms-key-2, settings.json, extensions/plugins, bin/lms, server-logs), regenerated on every start; the models it indexes live in lmstudio_models + nfs: false mounts: - service: lmstudio target: /root/.lmstudio -lmstudio_internal: +# nocheck: embedded-state-on-nfs content-addressed model blobs only; every writer fetches against the sha256 AI_LOCAL_MODELS declares and the play fails unless the store reads back at that digest, so concurrent writers of one file write identical bytes +lmstudio_models: type: volume - name: lmstudio_internal - # nocheck: nfs-false-data-loss Reason: regenerable per-daemon cache, rebuilt in full on first start against an empty volume; shared, a second daemon overwrites the auth key lms-key-2 and the first one's CLI is rejected - nfs: false - backup: false + name: lmstudio_models mounts: - service: lmstudio - target: /root/.lmstudio/.internal + target: /root/.lmstudio/models diff --git a/roles/svc-ai-lmstudio/tasks/00_core.yml b/roles/svc-ai-lmstudio/tasks/00_core.yml index 16942c364d..e29cf2a6f2 100644 --- a/roles/svc-ai-lmstudio/tasks/00_core.yml +++ b/roles/svc-ai-lmstudio/tasks/00_core.yml @@ -1,6 +1,14 @@ --- - include_tasks: utils/once/flag.yml +- name: "⬇️ Seed the shared model store before the daemons index it" + ansible.builtin.include_tasks: + file: utils/preseed.yml + apply: + delegate_to: "{{ LMSTUDIO_NFS_SERVER }}" + run_once: true + when: LMSTUDIO_PRESEED_ENABLED | bool + - name: "🔧 Setup container network for {{ application_id }}" include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/network/routine.yml') }}" vars: diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index ac3c91c989..a4362e07d8 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -1,26 +1,10 @@ --- -- name: "🔍 Resolve {{ application_id }} hosting node + container id" - ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}" - vars: - resolve_service_key: "{{ entity_name }}" - resolve_output_prefix: LMSTUDIO - resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" +- name: "🔍 Resolve every node that hosts an {{ application_id }} task" + ansible.builtin.include_tasks: utils/nodes.yml -- name: "🌱 Seed the models from the node cache before preloading" - ansible.builtin.include_tasks: - file: utils/seed.yml - apply: - delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - -- name: "⬇️ Preload the LM Studio models" - block: - - name: "⬇️ Preload the LM Studio models (first pass)" - ansible.builtin.include_tasks: utils/preload.yml - rescue: - - name: "🐝 Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and restarting the container it owns is the only lever the role has, though CI job 105558224377 shows a restart returning rc 0 without clearing it; the generic capture cannot even try, so swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises - ansible.builtin.fail: - msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" - when: DEPLOYMENT_MODE == 'swarm' - - - name: "♻️ Recover the wedged transfer and preload again" - ansible.builtin.include_tasks: utils/recover.yml +- name: "⬇️ Preload LM Studio on each hosting node" + ansible.builtin.include_tasks: utils/preload/node.yml + loop: "{{ LMSTUDIO_HOST_NODES }}" + loop_control: + loop_var: lmstudio_node + label: "{{ lmstudio_node }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/cid.yml b/roles/svc-ai-lmstudio/tasks/utils/cid.yml new file mode 100644 index 0000000000..e29b8fafba --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/cid.yml @@ -0,0 +1,22 @@ +--- +- name: "🔍 Resolve the LM Studio container on '{{ lmstudio_node }}'" + ansible.builtin.shell: | + {% if DEPLOYMENT_MODE == 'swarm' %} + container ps \ + --filter "label=com.docker.swarm.service.name={{ lmstudio_service_name }}" \ + --filter status=running \ + --format '{{ "{{.ID}}" }}' | head -1 + {% else %} + echo {{ LMSTUDIO_CONTAINER | quote }} + {% endif %} + register: lmstudio_node_cid + changed_when: false + retries: 30 + delay: 2 + until: DEPLOYMENT_MODE == 'compose' or (lmstudio_node_cid.stdout | trim) | length > 0 + delegate_to: "{{ lmstudio_node }}" + +- name: "🔧 Target '{{ lmstudio_node }}' for the next LM Studio step" + ansible.builtin.set_fact: + LMSTUDIO_HOST_NODE: "{{ lmstudio_node }}" + LMSTUDIO_LOCAL_CID: "{{ lmstudio_node_cid.stdout | trim }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/nodes.yml b/roles/svc-ai-lmstudio/tasks/utils/nodes.yml new file mode 100644 index 0000000000..b3ee1a6726 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/nodes.yml @@ -0,0 +1,29 @@ +--- +- name: "🔧 Resolve the swarm service name of '{{ entity_name }}'" + ansible.builtin.set_fact: + lmstudio_service_name: "{{ lookup('container_service', application_id, entity_name) if DEPLOYMENT_MODE == 'swarm' else '' }}" + +- name: "⏳ Wait until every LM Studio task has converged" + ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/wait_converged.yml') }}" + vars: + wait_service_key: "{{ entity_name }}" + wait_service_name: "{{ lmstudio_service_name }}" + +- name: "🔍 List every node that runs an LM Studio task" + ansible.builtin.shell: | + container service ps {{ lmstudio_service_name | quote }} \ + --filter desired-state=running \ + --format '{{ "{{.Node}}" }}' | sort -u + register: lmstudio_node_scan + changed_when: false + retries: 30 + delay: 2 + until: (lmstudio_node_scan.stdout | trim) | length > 0 + delegate_to: "{{ STACK_HOST }}" + when: DEPLOYMENT_MODE == 'swarm' + +- name: "🔧 Set the LM Studio hosting nodes" + ansible.builtin.set_fact: + LMSTUDIO_HOST_NODES: >- + {{ (lmstudio_node_scan.stdout_lines | map('trim') | select | list) + if DEPLOYMENT_MODE == 'swarm' else [inventory_hostname] }} diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml new file mode 100644 index 0000000000..fb48dfd0a6 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml @@ -0,0 +1,42 @@ +--- +- name: "🔍 Resolve the container before preloading on '{{ lmstudio_node }}'" + ansible.builtin.include_tasks: utils/cid.yml + +- name: "🗄️ Measure the space the model store offers on '{{ lmstudio_node }}'" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ LMSTUDIO_LOCAL_CID | quote }} df -P -B1 /root/.lmstudio/models + register: lmstudio_store_space + changed_when: false + retries: 30 + delay: 2 + until: lmstudio_store_space.rc == 0 + delegate_to: "{{ lmstudio_node }}" + when: not (LMSTUDIO_PRESEED_ENABLED | bool) + +- name: "🛑 Refuse to preload models the store on '{{ lmstudio_node }}' cannot hold" + vars: + _free: "{{ lmstudio_store_space.stdout_lines[1].split()[3] | int }}" + _required: "{{ LMSTUDIO_PRELOAD_LARGEST_BYTES | int }}" + ansible.builtin.fail: + msg: >- + The model store on {{ lmstudio_node }} has + {{ (_free | int / 1024 / 1024 / 1024) | round(1) }} GiB free, less than the + {{ (_required | int / 1024 / 1024 / 1024) | round(1) }} GiB the largest + declared model needs. Shrink AI_LOCAL_MODELS in group_vars/all/16_ai.yml or + give the node more disk. + when: + - not (LMSTUDIO_PRESEED_ENABLED | bool) + - _free | int < _required | int + +- name: "⬇️ Preload the LM Studio models on '{{ lmstudio_node }}'" + block: + - name: "⬇️ Preload the LM Studio models (first pass)" + ansible.builtin.include_tasks: utils/preload/pull.yml + rescue: + - name: "🐝 Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and restarting the container it owns is the only lever the role has, though CI job 105558224377 shows a restart returning rc 0 without clearing it; the generic capture cannot even try, so swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises + ansible.builtin.fail: + msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" + when: DEPLOYMENT_MODE == 'swarm' + + - name: "♻️ Recover the wedged transfer and preload again" + ansible.builtin.include_tasks: utils/preload/recover.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/pull.yml similarity index 100% rename from roles/svc-ai-lmstudio/tasks/utils/preload.yml rename to roles/svc-ai-lmstudio/tasks/utils/preload/pull.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/recover.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml similarity index 95% rename from roles/svc-ai-lmstudio/tasks/utils/recover.yml rename to roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml index cea5918d25..e8f526456f 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/recover.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml @@ -25,4 +25,4 @@ resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" - name: "⬇️ Preload the LM Studio models (after the restart)" - ansible.builtin.include_tasks: utils/preload.yml + ansible.builtin.include_tasks: utils/preload/pull.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preseed.yml b/roles/svc-ai-lmstudio/tasks/utils/preseed.yml new file mode 100644 index 0000000000..49fa570841 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preseed.yml @@ -0,0 +1,118 @@ +--- +- name: "📁 Create the shared model store before the daemons start" + ansible.builtin.file: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}" + state: directory + mode: "0755" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "📁 Create the staging directory beside the model store" + ansible.builtin.file: + path: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + state: directory + mode: "0755" + +- name: "🔍 Read what the shared model store already holds" + ansible.builtin.stat: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + checksum_algorithm: sha256 + register: lmstudio_present + changed_when: false + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "🗄️ Measure the free space the shared model store offers" + ansible.builtin.command: "df -P -B1 {{ LMSTUDIO_SHARED_MODELS_PATH | quote }}" + register: lmstudio_export_space + changed_when: false + +- name: "🛑 Refuse to fetch models the shared store cannot hold" + vars: + _free: "{{ lmstudio_export_space.stdout_lines[1].split()[3] | int }}" + _absent: >- + {{ lmstudio_present.results + | rejectattr('stat.exists') + | map(attribute='model.bytes') | map('int') | sum }} + _required: "{{ (_absent | int) + (LMSTUDIO_PRELOAD_LARGEST_BYTES | int) }}" + ansible.builtin.fail: + msg: >- + The shared model store on {{ LMSTUDIO_NFS_SERVER }} has + {{ (_free | int / 1024 / 1024 / 1024) | round(1) }} GiB free but needs + {{ (_required | int / 1024 / 1024 / 1024) | round(1) }} GiB: it is missing + {{ (_absent | int / 1024 / 1024 / 1024) | round(1) }} GiB of declared models, + plus {{ (LMSTUDIO_PRELOAD_LARGEST_BYTES | int / 1024 / 1024 / 1024) | round(1) }} + GiB for the largest one while it stages. Shrink AI_LOCAL_MODELS in + group_vars/all/16_ai.yml or grow the export. + when: _free | int < _required | int + +- name: "⬇️ Fetch the models into the shared store" + get_url_retry: + url: "{{ model.url }}" + dest: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + tmp_dest: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + checksum: "sha256:{{ model.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + register: lmstudio_fetch + changed_when: false + failed_when: false + +- name: "🪞 Fetch from a mirror when the source did not deliver" + get_url_retry: + url: "{{ item.1 }}" + dest: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ item.0.repo }}/{{ item.0.file }}" + tmp_dest: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + checksum: "sha256:{{ item.0.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True) }}" + loop_control: + label: "{{ item.0.file }} from {{ item.1 }}" + register: lmstudio_fetch_mirror + changed_when: false + failed_when: false + +- name: "🔍 Read back what the shared model store now holds" + ansible.builtin.stat: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + checksum_algorithm: sha256 + register: lmstudio_seeded + changed_when: false + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "🛑 Fail when no source delivered a declared model" + vars: + _missing: >- + {{ lmstudio_seeded.results + | rejectattr('stat.exists') + | map(attribute='model.file') | list }} + _said: >- + {{ (lmstudio_fetch.results | map(attribute='msg') | select | list) + + (lmstudio_fetch_mirror.results | default([]) | map(attribute='msg') | select | list) }} + ansible.builtin.fail: + msg: >- + The shared model store on {{ LMSTUDIO_NFS_SERVER }} is missing + {{ _missing | join(', ') }}. What the sources answered: + {{ _said | join(' | ') }} + when: _missing | length > 0 + +- name: "🛑 Fail when a seeded model does not match its checksum" + ansible.builtin.fail: + msg: >- + {{ item.model.file }} in the shared model store on {{ LMSTUDIO_NFS_SERVER }} + hashes to {{ item.stat.checksum }}, but AI_LOCAL_MODELS declares + {{ item.model.sha256 }}. + loop: "{{ lmstudio_seeded.results | selectattr('stat.exists') | list }}" + loop_control: + label: "{{ item.model.file }}" + when: item.stat.checksum != item.model.sha256 diff --git a/roles/svc-ai-lmstudio/tasks/utils/seed.yml b/roles/svc-ai-lmstudio/tasks/utils/seed.yml deleted file mode 100644 index 41d2501a3c..0000000000 --- a/roles/svc-ai-lmstudio/tasks/utils/seed.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -- name: "📁 Create the model cache directory on the hosting node" - ansible.builtin.file: - path: "{{ LMSTUDIO_MODEL_CACHE_DIR }}" - state: directory - mode: "0755" - -- name: "🗄️ Measure the free space the models have to fit into" - ansible.builtin.command: "df -P -B1 {{ LMSTUDIO_MODEL_CACHE_DIR | quote }}" - register: lmstudio_cache_space - changed_when: false - -- name: "🛑 Refuse to fetch models the hosting node cannot hold" - vars: - _free: "{{ lmstudio_cache_space.stdout_lines[1].split()[3] | int }}" - _required: "{{ LMSTUDIO_PRELOAD_REQUIRED_BYTES | int }}" - ansible.builtin.fail: - msg: >- - The declared LM Studio models need {{ (_required | int / 1024 / 1024 / 1024) - | round(1) }} GiB on {{ LMSTUDIO_HOST_NODE }} but only {{ (_free | int / 1024 - / 1024 / 1024) | round(1) }} GiB is free. That is - {{ LMSTUDIO_PRELOAD_COPIES_ON_DISK }} times the - {{ (LMSTUDIO_PRELOAD_BYTES | int / 1024 / 1024 / 1024) | round(1) }} GiB the - models declare, because the node cache, the staging copy inside the container - and the models volume all exist at once during the import. Shrink - AI_LOCAL_MODELS in group_vars/all/16_ai.yml or give the node more disk. - when: _free | int < _required | int - -- name: "⬇️ Warm the model cache on the node, best effort" - get_url_retry: - url: "{{ model.source }}/resolve/main/{{ model.file }}" - dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" - checksum: "sha256:{{ model.sha256 }}" - mode: "0644" - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.file }}" - changed_when: false - failed_when: false - -- name: "🪞 Warm the model cache from a mirror when the source stayed silent" - get_url_retry: - url: "{{ item.1 }}/resolve/main/{{ item.0.file }}" - dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ item.0.file }}" - checksum: "sha256:{{ item.0.sha256 }}" - mode: "0644" - loop: "{{ LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True) }}" - loop_control: - label: "{{ item.0.file }} from {{ item.1 }}" - changed_when: false - failed_when: false - -- name: "📋 Hand the warmed model to LM Studio" - ansible.builtin.script: shell/import.sh - environment: - LMSTUDIO_LOCAL_CID: "{{ LMSTUDIO_LOCAL_CID }}" - MODEL_FILE: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" - MODEL_USER_REPO: "{{ model.source | regex_replace('^https://huggingface\\.co/', '') }}" - MODELS_DIR: "{{ LMSTUDIO_MODELS_DIR }}" - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.file }}" - changed_when: false diff --git a/roles/svc-ai-lmstudio/templates/test.env.j2 b/roles/svc-ai-lmstudio/templates/test.env.j2 new file mode 100644 index 0000000000..4913d84187 --- /dev/null +++ b/roles/svc-ai-lmstudio/templates/test.env.j2 @@ -0,0 +1,5 @@ +LMSTUDIO_CONTAINER={{ CLI_LOCAL_CID }} +LMSTUDIO_EXPECTED_BLOBS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='repo') | zip(LMSTUDIO_PRELOAD_MODELS | map(attribute='file')) | map('join', '/') | list | to_json }}' +LMSTUDIO_EXPECTED_DIGESTS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='sha256') | list | to_json }}' +LMSTUDIO_EXPECTED_MODELS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='name') | list | to_json }}' +LMSTUDIO_PORT={{ LMSTUDIO_PORT }} diff --git a/roles/svc-ai-lmstudio/vars/main.yml b/roles/svc-ai-lmstudio/vars/main.yml index 3c55763554..b55d6d4644 100644 --- a/roles/svc-ai-lmstudio/vars/main.yml +++ b/roles/svc-ai-lmstudio/vars/main.yml @@ -8,12 +8,14 @@ LMSTUDIO_PRELOAD_MODELS: "{{ lookup('config', application_id, 'services.lmstudio LMSTUDIO_PRELOAD_ASYNC_TIME: "{{ lookup('config', application_id, 'services.lmstudio.preload_async_time') }}" LMSTUDIO_PRELOAD_FLOOR_SECONDS: "{{ lookup('config', application_id, 'services.lmstudio.preload_floor_seconds') }}" LMSTUDIO_PRELOAD_MIN_BYTES_PER_SECOND: "{{ lookup('config', application_id, 'services.lmstudio.preload_min_bytes_per_second') }}" -LMSTUDIO_PRELOAD_BYTES: >- - {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | sum }} -LMSTUDIO_PRELOAD_COPIES_ON_DISK: 3 -LMSTUDIO_PRELOAD_REQUIRED_BYTES: >- - {{ (LMSTUDIO_PRELOAD_BYTES | int) * (LMSTUDIO_PRELOAD_COPIES_ON_DISK | int) }} -LMSTUDIO_CONFIG_DIR: "{{ lookup('container', application_id, 'directories.config') }}" -LMSTUDIO_MODEL_CACHE_DIR: "{{ [ LMSTUDIO_CONFIG_DIR, 'models' ] | path_join }}" -LMSTUDIO_HOME_DIR: "{{ lookup('volume', application_id, 'lmstudio_data').target }}" -LMSTUDIO_MODELS_DIR: "{{ [ LMSTUDIO_HOME_DIR, 'models' ] | path_join }}" +LMSTUDIO_PRELOAD_LARGEST_BYTES: >- + {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | max }} +LMSTUDIO_NFS_SERVER: "{{ (groups['svc-storage-nfs-server'] | default([]))[0] | default(inventory_hostname) }}" +LMSTUDIO_SHARED_MODELS_PATH: "{{ [ STORAGE_NFS_STATE_PATH, lookup('volume', application_id, 'lmstudio_models').name ] | path_join }}" +LMSTUDIO_SHARED_STAGING_PATH: "{{ [ STORAGE_NFS_STATE_PATH, '.lmstudio-staging' ] | path_join }}" +LMSTUDIO_PRESEED_ENABLED: >- + {{ (DEPLOYMENT_MODE == 'swarm') + and (storage.backend == 'nfs') + and ((groups['svc-storage-nfs-server'] | default([])) | length > 0) + and (lookup('volume', application_id, 'lmstudio_models').nfs is not false) + and (application_id not in lookup('roles_with_placement', 'manager')) }} diff --git a/tests/external/roles/test_ai_local_models_downloadable.py b/tests/external/roles/test_ai_local_models_downloadable.py index cc44488f16..4f013d6788 100644 --- a/tests/external/roles/test_ai_local_models_downloadable.py +++ b/tests/external/roles/test_ai_local_models_downloadable.py @@ -41,25 +41,26 @@ def _models() -> list[dict]: return [entry for entry in (entries or []) if isinstance(entry, dict)] -def _repo_path(source: str) -> str: - """Return ``user/repo`` for a HuggingFace source, else the empty string.""" - if not source.startswith(_HUGGINGFACE_PREFIX): +def _repo_path(entry: dict) -> str: + """Return the HuggingFace ``user/repo`` this entry pins, else empty. + + Args: + entry: one AI_LOCAL_MODELS declaration. + """ + url = str(entry.get("url") or "") + if not url.startswith(_HUGGINGFACE_PREFIX): return "" - return source[len(_HUGGINGFACE_PREFIX) :].strip("/") + return str(entry.get("repo") or "").strip("/") def _download_url(entry: dict) -> str: - return f"{entry['source'].rstrip('/')}/resolve/main/{entry['file']}" + return str(entry["url"]) def _mirror_urls(entry: dict) -> list[str]: - """Every extra base a deploy falls back to, in declaration order.""" + """Every extra URL a deploy falls back to, in declaration order.""" mirrors = entry.get("mirrors") or [] - return [ - f"{str(base).rstrip('/')}/resolve/main/{entry['file']}" - for base in mirrors - if isinstance(base, str) - ] + return [str(url) for url in mirrors if isinstance(url, str)] def _upstream_digests(repo_path: str) -> tuple[str, object]: @@ -133,11 +134,14 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): unpinned = [] for entry in models: alias = str(entry.get("alias") or "") - source = entry.get("source") + url = entry.get("url") + repo = entry.get("repo") file_name = entry.get("file") digest = entry.get("sha256") - if not isinstance(source, str) or not urlsplit(source).scheme: - unpinned.append(f"{alias}: 'source' is not an absolute URL") + if not isinstance(url, str) or not urlsplit(url).scheme: + unpinned.append(f"{alias}: 'url' is not an absolute URL") + if not isinstance(repo, str) or len(repo.strip("/").split("/")) != 2: + unpinned.append(f"{alias}: 'repo' is not '/'") if not isinstance(file_name, str) or not file_name: unpinned.append(f"{alias}: 'file' is missing") if not isinstance(digest, str) or not _SHA256.match(digest): @@ -149,21 +153,21 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): self.assertFalse( unpinned, f"{len(unpinned)} model declaration(s) in {_AI_VARS_FILE} cannot be " - f"fetched with a checksum. Each entry needs 'source', 'file' and " - f"'sha256':\n" + "\n".join(f" {line}" for line in unpinned), + f"fetched with a checksum. Each entry needs 'url', 'repo', 'file' " + f"and 'sha256':\n" + "\n".join(f" {line}" for line in unpinned), ) def test_every_pin_matches_what_upstream_serves(self) -> None: models = [ entry for entry in _models() - if isinstance(entry.get("source"), str) + if isinstance(entry.get("url"), str) and isinstance(entry.get("file"), str) and isinstance(entry.get("sha256"), str) ] self.assertTrue(models, f"{_AI_VARS_FILE} declares no pinned model") - repo_paths = [_repo_path(entry["source"]) for entry in models] + repo_paths = [_repo_path(entry) for entry in models] with concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: api = list(pool.map(_upstream_digests, repo_paths)) reach = list(pool.map(_reachable, [_download_url(e) for e in models]))