diff --git a/group_vars/all/16_ai.yml b/group_vars/all/16_ai.yml index 8c47b2dcb1..c79faa45c5 100644 --- a/group_vars/all/16_ai.yml +++ b/group_vars/all/16_ai.yml @@ -24,6 +24,8 @@ LITELLM_LMSTUDIO_BACKEND: >- AI_LOCAL_MODELS: - alias: qwen2.5:0.5b name: qwen2.5-0.5b-instruct + repo: lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF + url: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-Q8_0.gguf source: https://huggingface.co/lmstudio-community/Qwen2.5-0.5B-Instruct-GGUF file: Qwen2.5-0.5B-Instruct-Q8_0.gguf sha256: "1d96e30253a76970864323987e3e77eeaf170312952fd61d18d343c25cfd6e52" diff --git a/roles/svc-ai-lmstudio/files/shell/import.sh b/roles/svc-ai-lmstudio/files/shell/import.sh deleted file mode 100644 index 5e9e848d40..0000000000 --- a/roles/svc-ai-lmstudio/files/shell/import.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# -# Hand a pre-fetched GGUF to LM Studio, idempotent. -# Runs ON THE HOSTING NODE (delegated); targets the local container via -# `container exec` / `container cp`. -# -# Required env, supplied by the calling Ansible task: -# LMSTUDIO_LOCAL_CID resolved container id, local to this node -# MODEL_FILE node-side path to the fetched .gguf -# MODEL_USER_REPO HuggingFace "user/repo" the file belongs to -# MODELS_DIR models folder inside the container, under the -# volume target declared in meta/volumes.yml -set -euo pipefail - -ADDRESS="${LMSTUDIO_LOCAL_CID:?LMSTUDIO_LOCAL_CID env var (container id) required}" -BASE="$(basename "${MODEL_FILE}")" -TARGET="${MODELS_DIR}/${MODEL_USER_REPO}/${BASE}" - -if container exec "$ADDRESS" test -f "${TARGET}"; then - echo "PRESENT:${BASE}" - exit 0 -fi - -if [ ! -f "${MODEL_FILE}" ]; then - echo "ABSENT:${BASE}" - exit 0 -fi - -# nocheck: container-cp - MODEL_FILE is fetched on the node this script runs on -container cp "${MODEL_FILE}" "${ADDRESS}:/tmp/${BASE}" - -printf 'y\n' | container exec -i "$ADDRESS" lms import "/tmp/${BASE}" --yes --copy --user-repo "${MODEL_USER_REPO}" -container exec "$ADDRESS" rm -f "/tmp/${BASE}" -echo "IMPORTED:${BASE}" diff --git a/roles/svc-ai-lmstudio/files/test/.nocheck b/roles/svc-ai-lmstudio/files/test/.nocheck new file mode 100644 index 0000000000..a0fdb0bf8c --- /dev/null +++ b/roles/svc-ai-lmstudio/files/test/.nocheck @@ -0,0 +1 @@ +# nocheck: role-files-layout - Role test harness with test.sh as its entry point; the CLI runner stages roles//files/test/ by path. diff --git a/roles/svc-ai-lmstudio/files/test/test.sh b/roles/svc-ai-lmstudio/files/test/test.sh new file mode 100644 index 0000000000..fc83bfc748 --- /dev/null +++ b/roles/svc-ai-lmstudio/files/test/test.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# +# Env (rendered into test.env from templates/test.env.j2): +# LMSTUDIO_CONTAINER resolved container id on this node +# LMSTUDIO_PORT the daemon's HTTP port +# LMSTUDIO_EXPECTED_MODELS JSON array of model names the daemon must serve +# LMSTUDIO_EXPECTED_BLOBS JSON array of / paths under models/ +# LMSTUDIO_EXPECTED_DIGESTS JSON array of sha256, aligned with the blobs +# RETRIES attempts (default 30) +# SLEEP_SECONDS wait between (default 10) + +set -uo pipefail + +RETRIES="${RETRIES:-30}" +SLEEP_SECONDS="${SLEEP_SECONDS:-10}" +MODELS_DIR=/root/.lmstudio/models + +[ -n "${LMSTUDIO_CONTAINER}" ] || { + echo "[FATAL] LMSTUDIO_CONTAINER unset; the harness did not resolve a container" >&2 + exit 2 +} + +in_container() { + container exec "${LMSTUDIO_CONTAINER}" "$@" +} + +json_items() { + python3 -c 'import json,sys; [print(x) for x in json.loads(sys.argv[1])]' "$1" +} + +serves_model() { + in_container curl -fsS --connect-timeout 5 --max-time 30 "http://127.0.0.1:${LMSTUDIO_PORT}/v1/models" 2>/dev/null | + python3 -c 'import json,sys; d=json.load(sys.stdin); print("\n".join(m.get("id","") for m in d.get("data",[])))' 2>/dev/null | + grep -qxF "$1" +} + +failed=0 + +echo "=== 1. the CLI authenticates against its own daemon ===" +attempt=1 +auth_out="" +while [ "${attempt}" -le "${RETRIES}" ]; do + if auth_out="$(in_container /app/.bundle/lms ls 2>&1)"; then + echo "[OK] lms ls authenticated" + break + fi + if printf '%s' "${auth_out}" | grep -q 'authPacket'; then + echo "[FAIL] the daemon rejected its own CLI: $(printf '%s' "${auth_out}" | grep -m1 authPacket)" >&2 + echo " a second daemon shares /root/.lmstudio and rewrote .internal/lms-key-2" >&2 + exit 1 + fi + attempt=$((attempt + 1)) + [ "${attempt}" -le "${RETRIES}" ] && sleep "${SLEEP_SECONDS}" +done +if [ "${attempt}" -gt "${RETRIES}" ]; then + echo "[FAIL] lms ls never succeeded after ${RETRIES} attempts: ${auth_out}" >&2 + exit 1 +fi + +echo "=== 2. every declared blob is in place with its declared digest ===" +mapfile -t blobs < <(json_items "${LMSTUDIO_EXPECTED_BLOBS}") +mapfile -t digests < <(json_items "${LMSTUDIO_EXPECTED_DIGESTS}") +for i in "${!blobs[@]}"; do + path="${MODELS_DIR}/${blobs[$i]}" + if ! in_container test -f "${path}"; then + echo "[FAIL] ${blobs[$i]} is not in the model store" >&2 + failed=1 + continue + fi + actual="$(in_container sha256sum "${path}" 2>/dev/null | awk '{print $1}')" + if [ "${actual}" != "${digests[$i]}" ]; then + echo "[FAIL] ${blobs[$i]} hashes to ${actual}, declared ${digests[$i]}" >&2 + failed=1 + continue + fi + echo "[OK] ${blobs[$i]} present and matching" +done + +echo "=== 3. the daemon serves every declared model ===" +mapfile -t models < <(json_items "${LMSTUDIO_EXPECTED_MODELS}") +for model in "${models[@]}"; do + attempt=1 + while [ "${attempt}" -le "${RETRIES}" ]; do + if serves_model "${model}"; then + echo "[OK] ${model} is served" + break + fi + attempt=$((attempt + 1)) + [ "${attempt}" -le "${RETRIES}" ] && sleep "${SLEEP_SECONDS}" + done + if [ "${attempt}" -gt "${RETRIES}" ]; then + echo "[FAIL] ${model} is on disk but /v1/models never listed it" >&2 + echo " the daemon did not index the store it was given at startup" >&2 + failed=1 + fi +done + +[ "${failed}" -eq 0 ] || exit 1 +echo "[OK] svc-ai-lmstudio serves every declared model from the seeded store" diff --git a/roles/svc-ai-lmstudio/meta/tests.yml b/roles/svc-ai-lmstudio/meta/tests.yml new file mode 100644 index 0000000000..712dbdf2f8 --- /dev/null +++ b/roles/svc-ai-lmstudio/meta/tests.yml @@ -0,0 +1,4 @@ +--- +cli: + container_service_key: lmstudio + timeout: 900 diff --git a/roles/svc-ai-lmstudio/meta/volumes.yml b/roles/svc-ai-lmstudio/meta/volumes.yml index ae7259ecdf..0250216548 100644 --- a/roles/svc-ai-lmstudio/meta/volumes.yml +++ b/roles/svc-ai-lmstudio/meta/volumes.yml @@ -1,17 +1,17 @@ --- -# nocheck: embedded-state-on-nfs model blobs written once on pull and read back afterwards lmstudio_data: type: volume + name: lmstudio_data + # nocheck: nfs-false-data-loss Reason: per-daemon state LM Studio rewrites in place (lms-key-2, settings.json, extensions/plugins, bin/lms, server-logs), regenerated on every start; the models it indexes live in lmstudio_models + nfs: false mounts: - service: lmstudio target: /root/.lmstudio -lmstudio_internal: +# nocheck: embedded-state-on-nfs content-addressed model blobs only; every writer fetches against the sha256 AI_LOCAL_MODELS declares and the play fails unless the store reads back at that digest, so concurrent writers of one file write identical bytes +lmstudio_models: type: volume - name: lmstudio_internal - # nocheck: nfs-false-data-loss Reason: regenerable per-daemon cache, rebuilt in full on first start against an empty volume; shared, a second daemon overwrites the auth key lms-key-2 and the first one's CLI is rejected - nfs: false - backup: false + name: lmstudio_models mounts: - service: lmstudio - target: /root/.lmstudio/.internal + target: /root/.lmstudio/models diff --git a/roles/svc-ai-lmstudio/tasks/00_core.yml b/roles/svc-ai-lmstudio/tasks/00_core.yml index 16942c364d..e29cf2a6f2 100644 --- a/roles/svc-ai-lmstudio/tasks/00_core.yml +++ b/roles/svc-ai-lmstudio/tasks/00_core.yml @@ -1,6 +1,14 @@ --- - include_tasks: utils/once/flag.yml +- name: "⬇️ Seed the shared model store before the daemons index it" + ansible.builtin.include_tasks: + file: utils/preseed.yml + apply: + delegate_to: "{{ LMSTUDIO_NFS_SERVER }}" + run_once: true + when: LMSTUDIO_PRESEED_ENABLED | bool + - name: "🔧 Setup container network for {{ application_id }}" include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/network/routine.yml') }}" vars: diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index ac3c91c989..a4362e07d8 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -1,26 +1,10 @@ --- -- name: "🔍 Resolve {{ application_id }} hosting node + container id" - ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}" - vars: - resolve_service_key: "{{ entity_name }}" - resolve_output_prefix: LMSTUDIO - resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" +- name: "🔍 Resolve every node that hosts an {{ application_id }} task" + ansible.builtin.include_tasks: utils/nodes.yml -- name: "🌱 Seed the models from the node cache before preloading" - ansible.builtin.include_tasks: - file: utils/seed.yml - apply: - delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - -- name: "⬇️ Preload the LM Studio models" - block: - - name: "⬇️ Preload the LM Studio models (first pass)" - ansible.builtin.include_tasks: utils/preload.yml - rescue: - - name: "🐝 Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and restarting the container it owns is the only lever the role has, though CI job 105558224377 shows a restart returning rc 0 without clearing it; the generic capture cannot even try, so swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises - ansible.builtin.fail: - msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" - when: DEPLOYMENT_MODE == 'swarm' - - - name: "♻️ Recover the wedged transfer and preload again" - ansible.builtin.include_tasks: utils/recover.yml +- name: "⬇️ Preload LM Studio on each hosting node" + ansible.builtin.include_tasks: utils/preload/node.yml + loop: "{{ LMSTUDIO_HOST_NODES }}" + loop_control: + loop_var: lmstudio_node + label: "{{ lmstudio_node }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/cid.yml b/roles/svc-ai-lmstudio/tasks/utils/cid.yml new file mode 100644 index 0000000000..e29b8fafba --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/cid.yml @@ -0,0 +1,22 @@ +--- +- name: "🔍 Resolve the LM Studio container on '{{ lmstudio_node }}'" + ansible.builtin.shell: | + {% if DEPLOYMENT_MODE == 'swarm' %} + container ps \ + --filter "label=com.docker.swarm.service.name={{ lmstudio_service_name }}" \ + --filter status=running \ + --format '{{ "{{.ID}}" }}' | head -1 + {% else %} + echo {{ LMSTUDIO_CONTAINER | quote }} + {% endif %} + register: lmstudio_node_cid + changed_when: false + retries: 30 + delay: 2 + until: DEPLOYMENT_MODE == 'compose' or (lmstudio_node_cid.stdout | trim) | length > 0 + delegate_to: "{{ lmstudio_node }}" + +- name: "🔧 Target '{{ lmstudio_node }}' for the next LM Studio step" + ansible.builtin.set_fact: + LMSTUDIO_HOST_NODE: "{{ lmstudio_node }}" + LMSTUDIO_LOCAL_CID: "{{ lmstudio_node_cid.stdout | trim }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/nodes.yml b/roles/svc-ai-lmstudio/tasks/utils/nodes.yml new file mode 100644 index 0000000000..b3ee1a6726 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/nodes.yml @@ -0,0 +1,29 @@ +--- +- name: "🔧 Resolve the swarm service name of '{{ entity_name }}'" + ansible.builtin.set_fact: + lmstudio_service_name: "{{ lookup('container_service', application_id, entity_name) if DEPLOYMENT_MODE == 'swarm' else '' }}" + +- name: "⏳ Wait until every LM Studio task has converged" + ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/wait_converged.yml') }}" + vars: + wait_service_key: "{{ entity_name }}" + wait_service_name: "{{ lmstudio_service_name }}" + +- name: "🔍 List every node that runs an LM Studio task" + ansible.builtin.shell: | + container service ps {{ lmstudio_service_name | quote }} \ + --filter desired-state=running \ + --format '{{ "{{.Node}}" }}' | sort -u + register: lmstudio_node_scan + changed_when: false + retries: 30 + delay: 2 + until: (lmstudio_node_scan.stdout | trim) | length > 0 + delegate_to: "{{ STACK_HOST }}" + when: DEPLOYMENT_MODE == 'swarm' + +- name: "🔧 Set the LM Studio hosting nodes" + ansible.builtin.set_fact: + LMSTUDIO_HOST_NODES: >- + {{ (lmstudio_node_scan.stdout_lines | map('trim') | select | list) + if DEPLOYMENT_MODE == 'swarm' else [inventory_hostname] }} diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml new file mode 100644 index 0000000000..fb48dfd0a6 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/node.yml @@ -0,0 +1,42 @@ +--- +- name: "🔍 Resolve the container before preloading on '{{ lmstudio_node }}'" + ansible.builtin.include_tasks: utils/cid.yml + +- name: "🗄️ Measure the space the model store offers on '{{ lmstudio_node }}'" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec {{ LMSTUDIO_LOCAL_CID | quote }} df -P -B1 /root/.lmstudio/models + register: lmstudio_store_space + changed_when: false + retries: 30 + delay: 2 + until: lmstudio_store_space.rc == 0 + delegate_to: "{{ lmstudio_node }}" + when: not (LMSTUDIO_PRESEED_ENABLED | bool) + +- name: "🛑 Refuse to preload models the store on '{{ lmstudio_node }}' cannot hold" + vars: + _free: "{{ lmstudio_store_space.stdout_lines[1].split()[3] | int }}" + _required: "{{ LMSTUDIO_PRELOAD_LARGEST_BYTES | int }}" + ansible.builtin.fail: + msg: >- + The model store on {{ lmstudio_node }} has + {{ (_free | int / 1024 / 1024 / 1024) | round(1) }} GiB free, less than the + {{ (_required | int / 1024 / 1024 / 1024) | round(1) }} GiB the largest + declared model needs. Shrink AI_LOCAL_MODELS in group_vars/all/16_ai.yml or + give the node more disk. + when: + - not (LMSTUDIO_PRESEED_ENABLED | bool) + - _free | int < _required | int + +- name: "⬇️ Preload the LM Studio models on '{{ lmstudio_node }}'" + block: + - name: "⬇️ Preload the LM Studio models (first pass)" + ansible.builtin.include_tasks: utils/preload/pull.yml + rescue: + - name: "🐝 Re-raise the preload failure unchanged in swarm" # nocheck: rescue-recovery a timed-out transfer leaves the daemon reporting 0.00% for every further pull, and restarting the container it owns is the only lever the role has, though CI job 105558224377 shows a restart returning rc 0 without clearing it; the generic capture cannot even try, so swarm re-raises here instead, because under any_errors_fatal a rescued host keeps running while every other host is already marked failed, and compose ends the block by preloading again so an unrecovered failure still re-raises + ansible.builtin.fail: + msg: "{{ ansible_failed_result.msg | default('LM Studio preload failed') }}" + when: DEPLOYMENT_MODE == 'swarm' + + - name: "♻️ Recover the wedged transfer and preload again" + ansible.builtin.include_tasks: utils/preload/recover.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/pull.yml similarity index 100% rename from roles/svc-ai-lmstudio/tasks/utils/preload.yml rename to roles/svc-ai-lmstudio/tasks/utils/preload/pull.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/recover.yml b/roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml similarity index 95% rename from roles/svc-ai-lmstudio/tasks/utils/recover.yml rename to roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml index cea5918d25..e8f526456f 100644 --- a/roles/svc-ai-lmstudio/tasks/utils/recover.yml +++ b/roles/svc-ai-lmstudio/tasks/utils/preload/recover.yml @@ -25,4 +25,4 @@ resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" - name: "⬇️ Preload the LM Studio models (after the restart)" - ansible.builtin.include_tasks: utils/preload.yml + ansible.builtin.include_tasks: utils/preload/pull.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preseed.yml b/roles/svc-ai-lmstudio/tasks/utils/preseed.yml new file mode 100644 index 0000000000..49fa570841 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preseed.yml @@ -0,0 +1,118 @@ +--- +- name: "📁 Create the shared model store before the daemons start" + ansible.builtin.file: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}" + state: directory + mode: "0755" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "📁 Create the staging directory beside the model store" + ansible.builtin.file: + path: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + state: directory + mode: "0755" + +- name: "🔍 Read what the shared model store already holds" + ansible.builtin.stat: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + checksum_algorithm: sha256 + register: lmstudio_present + changed_when: false + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "🗄️ Measure the free space the shared model store offers" + ansible.builtin.command: "df -P -B1 {{ LMSTUDIO_SHARED_MODELS_PATH | quote }}" + register: lmstudio_export_space + changed_when: false + +- name: "🛑 Refuse to fetch models the shared store cannot hold" + vars: + _free: "{{ lmstudio_export_space.stdout_lines[1].split()[3] | int }}" + _absent: >- + {{ lmstudio_present.results + | rejectattr('stat.exists') + | map(attribute='model.bytes') | map('int') | sum }} + _required: "{{ (_absent | int) + (LMSTUDIO_PRELOAD_LARGEST_BYTES | int) }}" + ansible.builtin.fail: + msg: >- + The shared model store on {{ LMSTUDIO_NFS_SERVER }} has + {{ (_free | int / 1024 / 1024 / 1024) | round(1) }} GiB free but needs + {{ (_required | int / 1024 / 1024 / 1024) | round(1) }} GiB: it is missing + {{ (_absent | int / 1024 / 1024 / 1024) | round(1) }} GiB of declared models, + plus {{ (LMSTUDIO_PRELOAD_LARGEST_BYTES | int / 1024 / 1024 / 1024) | round(1) }} + GiB for the largest one while it stages. Shrink AI_LOCAL_MODELS in + group_vars/all/16_ai.yml or grow the export. + when: _free | int < _required | int + +- name: "⬇️ Fetch the models into the shared store" + get_url_retry: + url: "{{ model.url }}" + dest: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + tmp_dest: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + checksum: "sha256:{{ model.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + register: lmstudio_fetch + changed_when: false + failed_when: false + +- name: "🪞 Fetch from a mirror when the source did not deliver" + get_url_retry: + url: "{{ item.1 }}" + dest: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ item.0.repo }}/{{ item.0.file }}" + tmp_dest: "{{ LMSTUDIO_SHARED_STAGING_PATH }}" + checksum: "sha256:{{ item.0.sha256 }}" + mode: "0644" + loop: "{{ LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True) }}" + loop_control: + label: "{{ item.0.file }} from {{ item.1 }}" + register: lmstudio_fetch_mirror + changed_when: false + failed_when: false + +- name: "🔍 Read back what the shared model store now holds" + ansible.builtin.stat: + path: "{{ LMSTUDIO_SHARED_MODELS_PATH }}/{{ model.repo }}/{{ model.file }}" + checksum_algorithm: sha256 + register: lmstudio_seeded + changed_when: false + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.file }}" + +- name: "🛑 Fail when no source delivered a declared model" + vars: + _missing: >- + {{ lmstudio_seeded.results + | rejectattr('stat.exists') + | map(attribute='model.file') | list }} + _said: >- + {{ (lmstudio_fetch.results | map(attribute='msg') | select | list) + + (lmstudio_fetch_mirror.results | default([]) | map(attribute='msg') | select | list) }} + ansible.builtin.fail: + msg: >- + The shared model store on {{ LMSTUDIO_NFS_SERVER }} is missing + {{ _missing | join(', ') }}. What the sources answered: + {{ _said | join(' | ') }} + when: _missing | length > 0 + +- name: "🛑 Fail when a seeded model does not match its checksum" + ansible.builtin.fail: + msg: >- + {{ item.model.file }} in the shared model store on {{ LMSTUDIO_NFS_SERVER }} + hashes to {{ item.stat.checksum }}, but AI_LOCAL_MODELS declares + {{ item.model.sha256 }}. + loop: "{{ lmstudio_seeded.results | selectattr('stat.exists') | list }}" + loop_control: + label: "{{ item.model.file }}" + when: item.stat.checksum != item.model.sha256 diff --git a/roles/svc-ai-lmstudio/tasks/utils/seed.yml b/roles/svc-ai-lmstudio/tasks/utils/seed.yml deleted file mode 100644 index 41d2501a3c..0000000000 --- a/roles/svc-ai-lmstudio/tasks/utils/seed.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -- name: "📁 Create the model cache directory on the hosting node" - ansible.builtin.file: - path: "{{ LMSTUDIO_MODEL_CACHE_DIR }}" - state: directory - mode: "0755" - -- name: "🗄️ Measure the free space the models have to fit into" - ansible.builtin.command: "df -P -B1 {{ LMSTUDIO_MODEL_CACHE_DIR | quote }}" - register: lmstudio_cache_space - changed_when: false - -- name: "🛑 Refuse to fetch models the hosting node cannot hold" - vars: - _free: "{{ lmstudio_cache_space.stdout_lines[1].split()[3] | int }}" - _required: "{{ LMSTUDIO_PRELOAD_REQUIRED_BYTES | int }}" - ansible.builtin.fail: - msg: >- - The declared LM Studio models need {{ (_required | int / 1024 / 1024 / 1024) - | round(1) }} GiB on {{ LMSTUDIO_HOST_NODE }} but only {{ (_free | int / 1024 - / 1024 / 1024) | round(1) }} GiB is free. That is - {{ LMSTUDIO_PRELOAD_COPIES_ON_DISK }} times the - {{ (LMSTUDIO_PRELOAD_BYTES | int / 1024 / 1024 / 1024) | round(1) }} GiB the - models declare, because the node cache, the staging copy inside the container - and the models volume all exist at once during the import. Shrink - AI_LOCAL_MODELS in group_vars/all/16_ai.yml or give the node more disk. - when: _free | int < _required | int - -- name: "⬇️ Warm the model cache on the node, best effort" - get_url_retry: - url: "{{ model.source }}/resolve/main/{{ model.file }}" - dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" - checksum: "sha256:{{ model.sha256 }}" - mode: "0644" - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.file }}" - changed_when: false - failed_when: false - -- name: "🪞 Warm the model cache from a mirror when the source stayed silent" - get_url_retry: - url: "{{ item.1 }}/resolve/main/{{ item.0.file }}" - dest: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ item.0.file }}" - checksum: "sha256:{{ item.0.sha256 }}" - mode: "0644" - loop: "{{ LMSTUDIO_PRELOAD_MODELS | subelements('mirrors', skip_missing=True) }}" - loop_control: - label: "{{ item.0.file }} from {{ item.1 }}" - changed_when: false - failed_when: false - -- name: "📋 Hand the warmed model to LM Studio" - ansible.builtin.script: shell/import.sh - environment: - LMSTUDIO_LOCAL_CID: "{{ LMSTUDIO_LOCAL_CID }}" - MODEL_FILE: "{{ LMSTUDIO_MODEL_CACHE_DIR }}/{{ model.file }}" - MODEL_USER_REPO: "{{ model.source | regex_replace('^https://huggingface\\.co/', '') }}" - MODELS_DIR: "{{ LMSTUDIO_MODELS_DIR }}" - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.file }}" - changed_when: false diff --git a/roles/svc-ai-lmstudio/templates/test.env.j2 b/roles/svc-ai-lmstudio/templates/test.env.j2 new file mode 100644 index 0000000000..4913d84187 --- /dev/null +++ b/roles/svc-ai-lmstudio/templates/test.env.j2 @@ -0,0 +1,5 @@ +LMSTUDIO_CONTAINER={{ CLI_LOCAL_CID }} +LMSTUDIO_EXPECTED_BLOBS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='repo') | zip(LMSTUDIO_PRELOAD_MODELS | map(attribute='file')) | map('join', '/') | list | to_json }}' +LMSTUDIO_EXPECTED_DIGESTS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='sha256') | list | to_json }}' +LMSTUDIO_EXPECTED_MODELS='{{ LMSTUDIO_PRELOAD_MODELS | map(attribute='name') | list | to_json }}' +LMSTUDIO_PORT={{ LMSTUDIO_PORT }} diff --git a/roles/svc-ai-lmstudio/vars/main.yml b/roles/svc-ai-lmstudio/vars/main.yml index 3c55763554..b55d6d4644 100644 --- a/roles/svc-ai-lmstudio/vars/main.yml +++ b/roles/svc-ai-lmstudio/vars/main.yml @@ -8,12 +8,14 @@ LMSTUDIO_PRELOAD_MODELS: "{{ lookup('config', application_id, 'services.lmstudio LMSTUDIO_PRELOAD_ASYNC_TIME: "{{ lookup('config', application_id, 'services.lmstudio.preload_async_time') }}" LMSTUDIO_PRELOAD_FLOOR_SECONDS: "{{ lookup('config', application_id, 'services.lmstudio.preload_floor_seconds') }}" LMSTUDIO_PRELOAD_MIN_BYTES_PER_SECOND: "{{ lookup('config', application_id, 'services.lmstudio.preload_min_bytes_per_second') }}" -LMSTUDIO_PRELOAD_BYTES: >- - {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | sum }} -LMSTUDIO_PRELOAD_COPIES_ON_DISK: 3 -LMSTUDIO_PRELOAD_REQUIRED_BYTES: >- - {{ (LMSTUDIO_PRELOAD_BYTES | int) * (LMSTUDIO_PRELOAD_COPIES_ON_DISK | int) }} -LMSTUDIO_CONFIG_DIR: "{{ lookup('container', application_id, 'directories.config') }}" -LMSTUDIO_MODEL_CACHE_DIR: "{{ [ LMSTUDIO_CONFIG_DIR, 'models' ] | path_join }}" -LMSTUDIO_HOME_DIR: "{{ lookup('volume', application_id, 'lmstudio_data').target }}" -LMSTUDIO_MODELS_DIR: "{{ [ LMSTUDIO_HOME_DIR, 'models' ] | path_join }}" +LMSTUDIO_PRELOAD_LARGEST_BYTES: >- + {{ LMSTUDIO_PRELOAD_MODELS | map(attribute='bytes') | map('int') | max }} +LMSTUDIO_NFS_SERVER: "{{ (groups['svc-storage-nfs-server'] | default([]))[0] | default(inventory_hostname) }}" +LMSTUDIO_SHARED_MODELS_PATH: "{{ [ STORAGE_NFS_STATE_PATH, lookup('volume', application_id, 'lmstudio_models').name ] | path_join }}" +LMSTUDIO_SHARED_STAGING_PATH: "{{ [ STORAGE_NFS_STATE_PATH, '.lmstudio-staging' ] | path_join }}" +LMSTUDIO_PRESEED_ENABLED: >- + {{ (DEPLOYMENT_MODE == 'swarm') + and (storage.backend == 'nfs') + and ((groups['svc-storage-nfs-server'] | default([])) | length > 0) + and (lookup('volume', application_id, 'lmstudio_models').nfs is not false) + and (application_id not in lookup('roles_with_placement', 'manager')) }} diff --git a/tests/external/roles/test_ai_local_models_downloadable.py b/tests/external/roles/test_ai_local_models_downloadable.py index cc44488f16..4f013d6788 100644 --- a/tests/external/roles/test_ai_local_models_downloadable.py +++ b/tests/external/roles/test_ai_local_models_downloadable.py @@ -41,25 +41,26 @@ def _models() -> list[dict]: return [entry for entry in (entries or []) if isinstance(entry, dict)] -def _repo_path(source: str) -> str: - """Return ``user/repo`` for a HuggingFace source, else the empty string.""" - if not source.startswith(_HUGGINGFACE_PREFIX): +def _repo_path(entry: dict) -> str: + """Return the HuggingFace ``user/repo`` this entry pins, else empty. + + Args: + entry: one AI_LOCAL_MODELS declaration. + """ + url = str(entry.get("url") or "") + if not url.startswith(_HUGGINGFACE_PREFIX): return "" - return source[len(_HUGGINGFACE_PREFIX) :].strip("/") + return str(entry.get("repo") or "").strip("/") def _download_url(entry: dict) -> str: - return f"{entry['source'].rstrip('/')}/resolve/main/{entry['file']}" + return str(entry["url"]) def _mirror_urls(entry: dict) -> list[str]: - """Every extra base a deploy falls back to, in declaration order.""" + """Every extra URL a deploy falls back to, in declaration order.""" mirrors = entry.get("mirrors") or [] - return [ - f"{str(base).rstrip('/')}/resolve/main/{entry['file']}" - for base in mirrors - if isinstance(base, str) - ] + return [str(url) for url in mirrors if isinstance(url, str)] def _upstream_digests(repo_path: str) -> tuple[str, object]: @@ -133,11 +134,14 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): unpinned = [] for entry in models: alias = str(entry.get("alias") or "") - source = entry.get("source") + url = entry.get("url") + repo = entry.get("repo") file_name = entry.get("file") digest = entry.get("sha256") - if not isinstance(source, str) or not urlsplit(source).scheme: - unpinned.append(f"{alias}: 'source' is not an absolute URL") + if not isinstance(url, str) or not urlsplit(url).scheme: + unpinned.append(f"{alias}: 'url' is not an absolute URL") + if not isinstance(repo, str) or len(repo.strip("/").split("/")) != 2: + unpinned.append(f"{alias}: 'repo' is not '/'") if not isinstance(file_name, str) or not file_name: unpinned.append(f"{alias}: 'file' is missing") if not isinstance(digest, str) or not _SHA256.match(digest): @@ -149,21 +153,21 @@ class TestAiLocalModelsDownloadable(unittest.TestCase): self.assertFalse( unpinned, f"{len(unpinned)} model declaration(s) in {_AI_VARS_FILE} cannot be " - f"fetched with a checksum. Each entry needs 'source', 'file' and " - f"'sha256':\n" + "\n".join(f" {line}" for line in unpinned), + f"fetched with a checksum. Each entry needs 'url', 'repo', 'file' " + f"and 'sha256':\n" + "\n".join(f" {line}" for line in unpinned), ) def test_every_pin_matches_what_upstream_serves(self) -> None: models = [ entry for entry in _models() - if isinstance(entry.get("source"), str) + if isinstance(entry.get("url"), str) and isinstance(entry.get("file"), str) and isinstance(entry.get("sha256"), str) ] self.assertTrue(models, f"{_AI_VARS_FILE} declares no pinned model") - repo_paths = [_repo_path(entry["source"]) for entry in models] + repo_paths = [_repo_path(entry) for entry in models] with concurrent.futures.ThreadPoolExecutor(max_workers=_MAX_WORKERS) as pool: api = list(pool.map(_upstream_digests, repo_paths)) reach = list(pool.map(_reachable, [_download_url(e) for e in models]))