fix(lmstudio): recover a wedged model preload instead of retrying into it

A preload that fails leaves the daemon unable to move bytes: measured in CI job 104622514450, attempt 1 reached 437.37 MB of 531.07 MB and died with 'Download failed: Timed-out', then all five retries sat at 0.00% for a full hour each until CONTAINER_EXEC_TIMEOUT killed the client. Six executions, 18796s, no progress.

Killing the client does not touch the daemon that owns the transfer, so a plain retry cannot help. The rescue restarts the container, which a local probe showed restores a working transfer, then re-resolves the CID because swarm may reschedule the restarted task under a fresh id, and preloads once more. Pull and reap moved into one included unit so the rescue also covers the async path, where the failure surfaces at the reap rather than at the pull. retries drops from 5 to 1, since retries N yields N+1 executions.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

roles/svc-ai-lmstudio/tasks/01_manager_ops.yml

@@ -6,36 +6,10 @@
     resolve_output_prefix: LMSTUDIO
     resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}"
 
-- name: ⬇️ Pre-pull LM Studio models
-  ansible.builtin.shell: > # nocheck: container-exec-resolver  runtime-resolved CID, delegated to the hosting node
-    container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }}
-  register: lmstudio_pull_result
-  loop: "{{ LMSTUDIO_PRELOAD_MODELS }}"
-  loop_control:
-    loop_var: model
-    label: "{{ model.alias }}"
-  # rationale: async; model downloads run overlapped, results reaped below before playwright
-  async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
-  poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"
-  retries: "{{ 0 if ASYNC_ENABLED | bool else 5 }}"
-  delay: 15
-  until: >
-    (lmstudio_pull_result.rc | default(0)) == 0 or
-    ('already downloaded' in (lmstudio_pull_result.stdout | default('')))
-  changed_when: false
-  failed_when: >-
-    (lmstudio_pull_result.failed | default(false))
-    and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default('')))
-  delegate_to: "{{ LMSTUDIO_HOST_NODE }}"
-
-- name: "⏳ Reap the LM Studio model pre-pulls"
-  ansible.builtin.include_tasks:
-    file: utils/async/reap.yml
-    apply:
-      delegate_to: "{{ LMSTUDIO_HOST_NODE }}"
-  vars:
-    async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}"
-    async_reap_label: "LM Studio model pre-pull"
-    async_reap_tolerate:
-      - "already downloaded"
-    ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}"
+- name: "⬇️ Preload the LM Studio models"
+  block:
+    - name: "⬇️ Preload the LM Studio models (first pass)"
+      ansible.builtin.include_tasks: utils/preload.yml
+  rescue:
+    - name: "♻️ Recover the wedged transfer and preload again"
+      ansible.builtin.include_tasks: utils/recover.yml

roles/svc-ai-lmstudio/tasks/utils/preload.yml

@@ -0,0 +1,33 @@
+---
+- name: ⬇️ Pre-pull LM Studio models
+  ansible.builtin.shell: > # nocheck: container-exec-resolver  runtime-resolved CID, delegated to the hosting node
+    container exec -i {{ LMSTUDIO_LOCAL_CID | quote }} lms get --yes --gguf {{ model.source | quote }}
+  register: lmstudio_pull_result
+  loop: "{{ LMSTUDIO_PRELOAD_MODELS }}"
+  loop_control:
+    loop_var: model
+    label: "{{ model.alias }}"
+  async: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME if ASYNC_ENABLED | bool else omit }}"
+  poll: "{{ ASYNC_POLL if ASYNC_ENABLED | bool else omit }}"
+  retries: "{{ 0 if ASYNC_ENABLED | bool else 1 }}"
+  delay: 15
+  until: >
+    (lmstudio_pull_result.rc | default(0)) == 0 or
+    ('already downloaded' in (lmstudio_pull_result.stdout | default('')))
+  changed_when: false
+  failed_when: >-
+    (lmstudio_pull_result.failed | default(false))
+    and 'already downloaded' not in ((lmstudio_pull_result.stdout | default('')) ~ (lmstudio_pull_result.stderr | default('')))
+  delegate_to: "{{ LMSTUDIO_HOST_NODE }}"
+
+- name: "⏳ Reap the LM Studio model pre-pulls"
+  ansible.builtin.include_tasks:
+    file: utils/async/reap.yml
+    apply:
+      delegate_to: "{{ LMSTUDIO_HOST_NODE }}"
+  vars:
+    async_reap_jobs: "{{ lmstudio_pull_result.results | default([]) }}"
+    async_reap_label: "LM Studio model pre-pull"
+    async_reap_tolerate:
+      - "already downloaded"
+    ASYNC_TIME: "{{ LMSTUDIO_PRELOAD_ASYNC_TIME }}"

roles/svc-ai-lmstudio/tasks/utils/recover.yml

@@ -0,0 +1,28 @@
+---
+- name: "🩺 Dump the LM Studio transfer record"
+  ansible.builtin.shell: > # nocheck: container-exec-resolver  runtime-resolved CID, delegated to the hosting node
+    container exec -i {{ LMSTUDIO_LOCAL_CID | quote }}
+    cat /root/.lmstudio/.internal/single-downloads-info.json
+  register: lmstudio_transfer_record
+  changed_when: false
+  failed_when: false
+  delegate_to: "{{ LMSTUDIO_HOST_NODE }}"
+
+- name: "🩺 Report the LM Studio transfer record"
+  ansible.builtin.debug:
+    var: lmstudio_transfer_record.stdout
+
+- name: "♻️ Restart LM Studio to release the wedged transfer"
+  ansible.builtin.command: container restart {{ LMSTUDIO_LOCAL_CID }}
+  changed_when: true
+  delegate_to: "{{ LMSTUDIO_HOST_NODE }}"
+
+- name: "🔍 Re-resolve the {{ application_id }} hosting node + container id"
+  ansible.builtin.include_tasks: "{{ lookup('path_absolute', 'roles/sys-svc-compose/tasks/utils/swarm/resolve_host_cid.yml') }}"
+  vars:
+    resolve_service_key: "{{ entity_name }}"
+    resolve_output_prefix: LMSTUDIO
+    resolve_compose_cid: "{{ LMSTUDIO_CONTAINER }}"
+
+- name: "⬇️ Preload the LM Studio models (after the restart)"
+  ansible.builtin.include_tasks: utils/preload.yml