diff --git a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml index 60f97c71e8..24a691460d 100644 --- a/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml +++ b/roles/svc-ai-lmstudio/tasks/01_manager_ops.yml @@ -6,36 +6,10 @@ resolve_output_prefix: LMSTUDIO resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" -- name: ⬇️ Pre-pull LM Studio models - ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node - container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }} - register: lmstudio_pull_result - loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" - loop_control: - loop_var: model - label: "{{ model.alias }}" - # rationale: async; model downloads run overlapped, results reaped below before playwright - async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" - poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" - retries: "{{ 0 if ASYNC_ENABLED | bool else 5 }}" - delay: 15 - until: > - (lmstudio_pull_result.rc | default(0)) == 0 or - ('already downloaded' in (lmstudio_pull_result.stdout | default(''))) - changed_when: false - failed_when: >- - (lmstudio_pull_result.failed | default(false)) - and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default(''))) - delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - -- name: "⏳ Reap the LM Studio model pre-pulls" - ansible.builtin.include_tasks: - file: utils/async/reap.yml - apply: - delegate_to: "{{ LMSTUDIO_HOST_NODE }}" - vars: - async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}" - async_reap_label: "LM Studio model pre-pull" - async_reap_tolerate: - - "already downloaded" - ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}" +- name: "⬇️ Preload the LM Studio models" + block: + - name: "⬇️ Preload the LM Studio models (first pass)" + ansible.builtin.include_tasks: utils/preload.yml + rescue: + - name: "♻️ Recover the wedged transfer and preload again" + ansible.builtin.include_tasks: utils/recover.yml diff --git a/roles/svc-ai-lmstudio/tasks/utils/preload.yml b/roles/svc-ai-lmstudio/tasks/utils/preload.yml new file mode 100644 index 0000000000..9c668a977a --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/preload.yml @@ -0,0 +1,33 @@ +--- +- name: ⬇️ Pre-pull LM Studio models + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }} + register: lmstudio_pull_result + loop: "{{ LMSTUDIO_PRELOAD_MODELS }}" + loop_control: + loop_var: model + label: "{{ model.alias }}" + async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}" + poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}" + retries: "{{ 0 if ASYNC_ENABLED | bool else 1 }}" + delay: 15 + until: > + (lmstudio_pull_result.rc | default(0)) == 0 or + ('already downloaded' in (lmstudio_pull_result.stdout | default(''))) + changed_when: false + failed_when: >- + (lmstudio_pull_result.failed | default(false)) + and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default(''))) + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + +- name: "⏳ Reap the LM Studio model pre-pulls" + ansible.builtin.include_tasks: + file: utils/async/reap.yml + apply: + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + vars: + async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}" + async_reap_label: "LM Studio model pre-pull" + async_reap_tolerate: + - "already downloaded" + ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}" diff --git a/roles/svc-ai-lmstudio/tasks/utils/recover.yml b/roles/svc-ai-lmstudio/tasks/utils/recover.yml new file mode 100644 index 0000000000..cea5918d25 --- /dev/null +++ b/roles/svc-ai-lmstudio/tasks/utils/recover.yml @@ -0,0 +1,28 @@ +--- +- name: "🩺 Dump the LM Studio transfer record" + ansible.builtin.shell: > # nocheck: container-exec-resolver runtime-resolved CID, delegated to the hosting node + container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} + cat /root/.lmstudio/.internal/single-downloads-info.json + register: lmstudio_transfer_record + changed_when: false + failed_when: false + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + +- name: "🩺 Report the LM Studio transfer record" + ansible.builtin.debug: + var: lmstudio_transfer_record.stdout + +- name: "♻️ Restart LM Studio to release the wedged transfer" + ansible.builtin.command: container restart {{ LMSTUDIO_LOCAL_CID }} + changed_when: true + delegate_to: "{{ LMSTUDIO_HOST_NODE }}" + +- name: "🔍 Re-resolve the {{ application_id }} hosting node + container id" + ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}" + vars: + resolve_service_key: "{{ entity_name }}" + resolve_output_prefix: LMSTUDIO + resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}" + +- name: "⬇️ Preload the LM Studio models (after the restart)" + ansible.builtin.include_tasks: utils/preload.yml