From 567b1365c09960bd32df8116cb9e7ac88b85a3bd Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Wed, 13 Aug 2025 16:13:00 +0200 Subject: [PATCH] Nextcloud: async overhaul & task refactor (conditional wait, faster polling) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Add config.performance.async.wait_for and expose as nextcloud_wait_for_async_enabled to toggle waiting for async jobs. • Split system/admin/index maintenance into separate tasks: 02_add_missing_indices.yml, 03_admin.yml, 04_system_config.yml. • Refactor plugin flow: rename 02_plugin→05_plugin, 03_plugin_routines→06_plugin_routines, 04_plugin_enable_and_configure→07_plugin_enable_and_configure; remove old 03_plugin_routines and 05_system. • Harden async handling: filter async_status loops by ansible_job_id; conditionally wait only when nextcloud_wait_for_async_enabled; reduce delay to 1s. • Reorder main.yml to run system steps before plugin setup; keep handlers flush earlier. • env.j2: simplify get_app_conf lookups (drop extra True flag). • vars/main.yml: add nextcloud_host_nginx_path and nextcloud_wait_for_async_enabled. https://chatgpt.com/share/689c9d4a-1748-800f-b490-06a5a48dd831 --- roles/web-app-nextcloud/config/main.yml | 2 + .../tasks/02_add_missing_indices.yml | 19 ++++++ roles/web-app-nextcloud/tasks/03_admin.yml | 11 ++++ .../tasks/03_plugin_routines.yml | 31 ---------- .../tasks/04_system_config.yml | 34 +++++++++++ .../tasks/{02_plugin.yml => 05_plugin.yml} | 2 +- roles/web-app-nextcloud/tasks/05_system.yml | 8 --- .../tasks/06_plugin_routines.yml | 59 +++++++++++++++++++ ...yml => 07_plugin_enable_and_configure.yml} | 38 ++++++++++-- roles/web-app-nextcloud/tasks/main.yml | 54 +++++++---------- roles/web-app-nextcloud/templates/env.j2 | 10 ++-- roles/web-app-nextcloud/vars/main.yml | 12 ++-- 12 files changed, 192 insertions(+), 88 deletions(-) create mode 100644 roles/web-app-nextcloud/tasks/02_add_missing_indices.yml create mode 100644 roles/web-app-nextcloud/tasks/03_admin.yml delete mode 100644 roles/web-app-nextcloud/tasks/03_plugin_routines.yml create mode 100644 roles/web-app-nextcloud/tasks/04_system_config.yml rename roles/web-app-nextcloud/tasks/{02_plugin.yml => 05_plugin.yml} (88%) delete mode 100644 roles/web-app-nextcloud/tasks/05_system.yml create mode 100644 roles/web-app-nextcloud/tasks/06_plugin_routines.yml rename roles/web-app-nextcloud/tasks/{04_plugin_enable_and_configure.yml => 07_plugin_enable_and_configure.yml} (51%) diff --git a/roles/web-app-nextcloud/config/main.yml b/roles/web-app-nextcloud/config/main.yml index 48083764..96b085ef 100644 --- a/roles/web-app-nextcloud/config/main.yml +++ b/roles/web-app-nextcloud/config/main.yml @@ -70,6 +70,8 @@ performance: memory_limit: "{{ ((ansible_memtotal_mb | int) / 30)|int }}M" # Dynamic set memory limit upload_limit: "5G" # Set upload limit to 5GB for big media files opcache_memory_consumption: "{{ ((ansible_memtotal_mb | int) / 30)|int }}M" # Dynamic set memory consumption + async: + wait_for: "{{ enable_debug }}" # If debug is enabled wait_for async jobs plugins_enabled: true # Implemented for speeding up testing and debugging process. For productive environments keep it true and steer the apps via the plugins config diff --git a/roles/web-app-nextcloud/tasks/02_add_missing_indices.yml b/roles/web-app-nextcloud/tasks/02_add_missing_indices.yml new file mode 100644 index 00000000..304fd65b --- /dev/null +++ b/roles/web-app-nextcloud/tasks/02_add_missing_indices.yml @@ -0,0 +1,19 @@ +- name: "Launch async: add missing DB indices in Nextcloud" + ansible.builtin.command: > + {{ nextcloud_docker_exec_occ }} db:add-missing-indices + async: 3600 + poll: 0 + register: db_indices_job + +- name: "Wait for DB indices job" + ansible.builtin.async_status: + jid: "{{ db_indices_job.ansible_job_id }}" + register: db_indices_result + until: db_indices_result.finished + retries: 600 + delay: 1 + failed_when: db_indices_result.rc != 0 + changed_when: > + ('Adding additional' in (db_indices_result.stdout | default(''))) or + ('Removing' in (db_indices_result.stdout | default(''))) or + ('updated successfully' in (db_indices_result.stdout | default(''))) \ No newline at end of file diff --git a/roles/web-app-nextcloud/tasks/03_admin.yml b/roles/web-app-nextcloud/tasks/03_admin.yml new file mode 100644 index 00000000..69f0d9c8 --- /dev/null +++ b/roles/web-app-nextcloud/tasks/03_admin.yml @@ -0,0 +1,11 @@ +- name: Ensure Nextcloud administrator is in the 'admin' group + command: > + docker exec -u {{ nextcloud_docker_user }} {{ nextcloud_container }} + php occ group:adduser admin {{ nextcloud_administrator_username }} + register: add_admin_to_group + changed_when: "'Added user' in (add_admin_to_group.stdout | default(''))" + failed_when: > + (add_admin_to_group.rc != 0) and + ("is already a member of" not in ( + (add_admin_to_group.stderr | default('')) ~ (add_admin_to_group.stdout | default('')) + )) \ No newline at end of file diff --git a/roles/web-app-nextcloud/tasks/03_plugin_routines.yml b/roles/web-app-nextcloud/tasks/03_plugin_routines.yml deleted file mode 100644 index 6f3119a3..00000000 --- a/roles/web-app-nextcloud/tasks/03_plugin_routines.yml +++ /dev/null @@ -1,31 +0,0 @@ -- name: "Disable incompatible plugins for {{plugin_key}}." - command: "{{nextcloud_docker_exec_occ}} app:disable {{incompatible_plugin}}" - loop: "{{plugin_value.incompatible_plugins}}" - loop_control: - loop_var: incompatible_plugin - register: disable_incompatible_plugin_result - changed_when: disable_incompatible_plugin_result.rc == 0 and ("No such app enabled" not in disable_incompatible_plugin_result.stdout) - when: - - plugin_value.incompatible_plugins is defined - - plugin_value.incompatible_plugins | length > 0 - -- name: install {{ plugin_key }} nextcloud plugin - command: "{{ nextcloud_docker_exec_occ }} app:install {{ plugin_key }}" - register: install_result - failed_when: > - install_result.rc != 0 - and - ("already installed" not in install_result.stdout) - and - ("not compatible with this version of the server" not in install_result.stdout) - changed_when: > - install_result.rc == 0 - and - ("already installed" not in install_result.stdout) - -- include_tasks: 04_plugin_enable_and_configure.yml - when: - - install_result is defined - - > - install_result.rc == 0 - or "already installed" in install_result.stdout \ No newline at end of file diff --git a/roles/web-app-nextcloud/tasks/04_system_config.yml b/roles/web-app-nextcloud/tasks/04_system_config.yml new file mode 100644 index 00000000..68215b3e --- /dev/null +++ b/roles/web-app-nextcloud/tasks/04_system_config.yml @@ -0,0 +1,34 @@ +- name: Load System Nextcloud configuration variables + include_vars: + file: system.yml + +- name: "Launch async: apply Nextcloud system configs" + ansible.builtin.command: > + {{ nextcloud_docker_exec_occ }} + config:system:set {{ item.parameter }} + {% if item.type is defined %} --type {{ item.type }}{% endif %} + --value {{ item.value }} + loop: "{{ nextcloud_system_config }}" + loop_control: + label: "{{ item.parameter }}" + async: 300 + poll: 0 + register: syscfg_jobs + +- name: "Wait for system config jobs" + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ syscfg_jobs.results | default([]) }}" + loop_control: + label: "{{ item._ansible_item_label | default(item.item.parameter) }}" + register: syscfg_wait + until: syscfg_wait.finished + retries: 100 + delay: 1 + failed_when: > + (syscfg_wait.rc is defined and syscfg_wait.rc|int != 0) + changed_when: > + (syscfg_wait.stdout is defined) and + ("Value not changed" not in syscfg_wait.stdout) + when: + - nextcloud_wait_for_async_enabled | bool diff --git a/roles/web-app-nextcloud/tasks/02_plugin.yml b/roles/web-app-nextcloud/tasks/05_plugin.yml similarity index 88% rename from roles/web-app-nextcloud/tasks/02_plugin.yml rename to roles/web-app-nextcloud/tasks/05_plugin.yml index 80e1626e..c9b5883b 100644 --- a/roles/web-app-nextcloud/tasks/02_plugin.yml +++ b/roles/web-app-nextcloud/tasks/05_plugin.yml @@ -1,5 +1,5 @@ - block: - - include_tasks: 03_plugin_routines.yml + - include_tasks: 06_plugin_routines.yml when: plugin_value.enabled | bool - name: disable {{ plugin_key }} nextcloud plugin diff --git a/roles/web-app-nextcloud/tasks/05_system.yml b/roles/web-app-nextcloud/tasks/05_system.yml deleted file mode 100644 index 7ae7786f..00000000 --- a/roles/web-app-nextcloud/tasks/05_system.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: Load System Nextcloud configuration variables - include_vars: - file: system.yml - -- name: Apply Nextcloud configurations - loop: "{{ nextcloud_system_config }}" - command: "{{nextcloud_docker_exec_occ}} config:system:set {{ item.parameter }}{% if item.type is defined %} --type {{ item.type }}{% endif %} --value {{ item.value }}" - # No good changed_when condition available \ No newline at end of file diff --git a/roles/web-app-nextcloud/tasks/06_plugin_routines.yml b/roles/web-app-nextcloud/tasks/06_plugin_routines.yml new file mode 100644 index 00000000..4d486763 --- /dev/null +++ b/roles/web-app-nextcloud/tasks/06_plugin_routines.yml @@ -0,0 +1,59 @@ +# roles/web-app-nextcloud/tasks/06_plugin_routines.yml +- name: "Launch async: disable incompatible plugins for {{ plugin_key }}" + ansible.builtin.command: "{{ nextcloud_docker_exec_occ }} app:disable {{ incompatible_plugin }}" + loop: "{{ plugin_value.incompatible_plugins }}" + loop_control: + loop_var: incompatible_plugin + label: "{{ incompatible_plugin }}" + when: + - plugin_value.incompatible_plugins is defined + - plugin_value.incompatible_plugins | length > 0 + async: 180 + poll: 0 + register: disable_incompat_jobs + +- name: "Wait for disable jobs" + vars: + jobs_with_id: >- + {{ (disable_incompat_jobs.results | default([])) + | selectattr('ansible_job_id','defined') + | list }} + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + loop: "{{ jobs_with_id }}" + loop_control: + label: "{{ item._ansible_item_label }}" + register: disable_incompat_wait + until: disable_incompat_wait.finished + retries: 100 + delay: 1 + when: + - jobs_with_id | length > 0 + - nextcloud_wait_for_async_enabled | bool + failed_when: > + (disable_incompat_wait.rc is defined and disable_incompat_wait.rc|int != 0) + and ('No such app enabled' not in (disable_incompat_wait.stdout | default(''))) + changed_when: > + (disable_incompat_wait.rc | default(0) | int == 0) + and ('No such app enabled' not in (disable_incompat_wait.stdout | default(''))) + +- name: install {{ plugin_key }} nextcloud plugin + command: "{{ nextcloud_docker_exec_occ }} app:install {{ plugin_key }}" + register: install_result + failed_when: > + install_result.rc != 0 + and + ("already installed" not in install_result.stdout) + and + ("not compatible with this version of the server" not in install_result.stdout) + changed_when: > + install_result.rc == 0 + and + ("already installed" not in install_result.stdout) + +- include_tasks: 07_plugin_enable_and_configure.yml + when: + - install_result is defined + - > + install_result.rc == 0 + or "already installed" in install_result.stdout \ No newline at end of file diff --git a/roles/web-app-nextcloud/tasks/04_plugin_enable_and_configure.yml b/roles/web-app-nextcloud/tasks/07_plugin_enable_and_configure.yml similarity index 51% rename from roles/web-app-nextcloud/tasks/04_plugin_enable_and_configure.yml rename to roles/web-app-nextcloud/tasks/07_plugin_enable_and_configure.yml index 93024893..199c3b09 100644 --- a/roles/web-app-nextcloud/tasks/04_plugin_enable_and_configure.yml +++ b/roles/web-app-nextcloud/tasks/07_plugin_enable_and_configure.yml @@ -15,13 +15,41 @@ file: "{{nextcloud_control_node_plugin_vars_directory}}{{ plugin_key }}.yml" when: plugin_vars_file.stat.exists -- name: "Set {{ item.configkey }} for {{ item.appid }}" +- name: "Launch async: set {{ item.configkey }} for {{ item.appid }}" + ansible.builtin.command: > + {{ nextcloud_docker_exec_occ }} config:app:set {{ item.appid }} {{ item.configkey }} + --value '{{ item.configvalue | to_json if item.configvalue is mapping else item.configvalue }}' loop: "{{ plugin_configuration }}" - command: > - {{ nextcloud_docker_exec_occ }} config:app:set {{ item.appid }} {{ item.configkey }} --value '{{ item.configvalue | to_json if item.configvalue is mapping else item.configvalue }}' - register: config_set_result - changed_when: (config_set_result.stdout is defined) and ("Config value were not updated" not in config_set_result.stdout) + loop_control: + label: "{{ item.appid }}:{{ item.configkey }}" when: plugin_vars_file.stat.exists + async: 300 # max runtime per call (seconds) — adjust as needed + poll: 0 # don't wait; run in background + register: config_set_jobs + +- name: "Wait for async jobs" + vars: + jobs_with_id: >- + {{ (config_set_jobs.results | default([])) + | selectattr('ansible_job_id','defined') + | list }} + ansible.builtin.async_status: + jid: "{{ item.ansible_job_id }}" + register: config_set_wait + until: config_set_wait.finished + retries: 100 + delay: 1 + loop: "{{ jobs_with_id }}" + loop_control: + label: "{{ item._ansible_item_label | default(item.item.appid ~ ':' ~ item.item.configkey) }}" + when: + - jobs_with_id | length > 0 + - nextcloud_wait_for_async_enabled | bool + failed_when: > + (config_set_wait.rc is defined and config_set_wait.rc|int != 0) + changed_when: > + (config_set_wait.stdout is defined) and + ("Config value were not updated" not in config_set_wait.stdout) - name: Check if {{nextcloud_control_node_plugin_tasks_directory}}{{ plugin_key }}.yml exists stat: diff --git a/roles/web-app-nextcloud/tasks/main.yml b/roles/web-app-nextcloud/tasks/main.yml index df5a4174..6d57fa5a 100644 --- a/roles/web-app-nextcloud/tasks/main.yml +++ b/roles/web-app-nextcloud/tasks/main.yml @@ -1,7 +1,19 @@ --- +- name: "include role for {{application_id}} to receive certs & do modification routines" + include_role: + name: srv-web-7-6-composer + +- name: create nextcloud proxy configuration file + template: + src: "nginx/host.conf.j2" + dest: "{{ nextcloud_host_nginx_path }}" + notify: restart openresty + - name: "load docker and db for {{application_id}}" include_role: name: cmp-db-docker + vars: + docker_compose_flush_handlers: false - name: "create {{ nextcloud_host_config_additives_directory }}" file: @@ -13,22 +25,12 @@ template: src: "{{ item }}" dest: "{{ nextcloud_host_config_additives_directory }}/{{ item | basename | regex_replace('\\.j2$', '') }}" - owner: "{{nextcloud_docker_user_id}}" - group: "{{nextcloud_docker_user_id}}" + owner: "{{ nextcloud_docker_user_id }}" + group: "{{ nextcloud_docker_user_id }}" loop: "{{ lookup('fileglob', role_path ~ '/templates/config/*.j2', wantlist=True) }}" # Not all type of changes take instantly place. Due to this reason a rebuild is required. notify: docker compose up -- name: "include role for {{application_id}} to receive certs & do modification routines" - include_role: - name: srv-web-7-6-composer - -- name: create nextcloud proxy configuration file - template: - src: "nginx/host.conf.j2" - dest: "{{nginx.directories.http.servers}}{{domains | get_domain(application_id)}}.conf" - notify: restart openresty - - name: create internal nextcloud nginx configuration template: src: "nginx/docker.conf.j2" @@ -41,8 +43,15 @@ - name: Flush all handlers immediately so that occ can be used meta: flush_handlers +- name: Load system configuration steps + include_tasks: "{{ item }}" + loop: + - 02_add_missing_indices.yml + - 03_admin.yml + - 04_system_config.yml + - name: Setup Nextcloud Plugins - include_tasks: 02_plugin.yml + include_tasks: 05_plugin.yml loop: "{{ applications | get_app_conf(application_id, 'plugins', True) | dict2items }}" loop_control: loop_var: plugin_item @@ -51,23 +60,4 @@ plugin_value: "{{ plugin_item.value }}" when: nextcloud_plugins_enabled -- name: Load system configuration - include_tasks: 05_system.yml -- name: Add missing database indices in Nextcloud - command: > - {{ nextcloud_docker_exec_occ }} db:add-missing-indices - register: db_indices_result - changed_when: > - 'Adding additional' in db_indices_result.stdout or - 'Removing' in db_indices_result.stdout or - 'updated successfully' in db_indices_result.stdout - failed_when: db_indices_result.rc != 0 - -- name: Ensure Nextcloud administrator is in the 'admin' group - command: > - docker exec -u {{ nextcloud_docker_user }} {{ nextcloud_container }} - php occ group:adduser admin {{ nextcloud_administrator_username }} - register: add_admin_to_group - changed_when: "'Added user' in add_admin_to_group.stdout" - failed_when: add_admin_to_group.rc != 0 and "'is already a member of' not in add_admin_to_group.stderr" diff --git a/roles/web-app-nextcloud/templates/env.j2 b/roles/web-app-nextcloud/templates/env.j2 index e57bbd40..dee5082d 100644 --- a/roles/web-app-nextcloud/templates/env.j2 +++ b/roles/web-app-nextcloud/templates/env.j2 @@ -8,9 +8,9 @@ MYSQL_PASSWORD= "{{database_password}}" MYSQL_HOST= "{{database_host}}:{{database_port}}" # PHP -PHP_MEMORY_LIMIT= "{{applications | get_app_conf(application_id, 'performance.php.memory_limit', True)}}" -PHP_UPLOAD_LIMIT= "{{applications | get_app_conf(application_id, 'performance.php.upload_limit', True)}}" -PHP_OPCACHE_MEMORY_CONSUMPTION= "{{applications | get_app_conf(application_id, 'performance.php.opcache_memory_consumption', True)}}" +PHP_MEMORY_LIMIT= "{{applications | get_app_conf(application_id, 'performance.php.memory_limit')}}" +PHP_UPLOAD_LIMIT= "{{applications | get_app_conf(application_id, 'performance.php.upload_limit')}}" +PHP_OPCACHE_MEMORY_CONSUMPTION= "{{applications | get_app_conf(application_id, 'performance.php.opcache_memory_consumption')}}" # Email Configuration SMTP_HOST= {{system_email.host}} @@ -24,8 +24,8 @@ MAIL_FROM_ADDRESS= "{{ users['no-reply'].username }}" MAIL_DOMAIN= "{{ system_email.domain }}" # Initial Admin Data -NEXTCLOUD_ADMIN_USER= "{{applications | get_app_conf(application_id, 'users.administrator.username', True)}}" -NEXTCLOUD_ADMIN_PASSWORD= "{{applications | get_app_conf(application_id, 'credentials.administrator_password', True)}}" +NEXTCLOUD_ADMIN_USER= "{{applications | get_app_conf(application_id, 'users.administrator.username')}}" +NEXTCLOUD_ADMIN_PASSWORD= "{{applications | get_app_conf(application_id, 'credentials.administrator_password')}}" # Security diff --git a/roles/web-app-nextcloud/vars/main.yml b/roles/web-app-nextcloud/vars/main.yml index 4eaba9cf..c910fd53 100644 --- a/roles/web-app-nextcloud/vars/main.yml +++ b/roles/web-app-nextcloud/vars/main.yml @@ -9,12 +9,11 @@ http_port: "{{ ports.localhost.http[applica # Database database_password: "{{ applications | get_app_conf(application_id, 'credentials.database_password', True)}}" -database_type: "mariadb" # Database flavor +database_type: "mariadb" # Database flavor + +nextcloud_wait_for_async_enabled: "{{applications | get_app_conf(application_id, 'performance.async.wait_for')}}" + nextcloud_plugins_enabled: "{{ applications | get_app_conf(application_id, 'plugins_enabled', True) }}" - -# Docker -docker_compose_flush_handlers: false # Deactivate flushing because first some routines have to be done - nextcloud_administrator_username: "{{ applications | get_app_conf(application_id, 'users.administrator.username', True) }}" # Control Node @@ -24,8 +23,9 @@ nextcloud_control_node_plugin_tasks_directory: "{{role_path}}/tasks/plugins/" # Host ## Host Paths -nextcloud_host_config_additives_directory: "{{ docker_compose.directories.volumes }}infinito/" # This folder is the path to which the additive configurations will be copied +nextcloud_host_config_additives_directory: "{{ docker_compose.directories.volumes }}infinito/" # This folder is the path to which the additive configurations will be copied nextcloud_host_include_instructions_file: "{{ docker_compose.directories.volumes }}includes.php" # Path to the instruction file on the host. Responsible for loading the additional configurations +nextcloud_host_nginx_path: "{{ nginx.directories.http.servers }}{{ domains | get_domain(application_id) }}.conf" # Nginx path for proxy conf # Docker