From 2d80ad5f1b1cd1652cdd4dfa2c332ba7a9419970 Mon Sep 17 00:00:00 2001 From: Kevin Veen-Birkenbach Date: Fri, 30 Oct 2026 16:51:56 +0100 Subject: [PATCH] fix(web-app-openproject): wait for the swarm seeder and detect dead DB sockets Compose gates web, worker and cron on the seeder via depends_on service_completed_successfully. Swarm ignores depends_on, so nothing waited for the one-shot seeder there. In CI (store-i18n run 40979252521, OpenProject#0 swarm) an overlay flap reset the seeder's Postgres connection; Postgres rolled back RootSeeder's transaction, but the seeder never noticed and sat idle on the dead socket forever ("Executing database seed..."). No UserCustomFieldSection rows were committed, so /my/account rendered without fieldsets and the deploy still reported green. - 01_manager_ops.yml: in swarm, after the handler flush deploys the stack, poll job_state.sh on the seeder service until its newest task is Complete (fail on Shutdown, give up after 20 minutes), mirroring web-app-zammad's init-job wait. Compose keeps relying on depends_on. - env.j2: add libpq TCP keepalives (idle 60s, interval 10s, count 6) and tcp_user_timeout=120000 to DATABASE_URL, so a connection whose peer vanished errors out within about two minutes instead of hanging. The seeder then exits non-zero and swarm's on-failure restart policy reruns the seed. Co-Authored-By: Claude Opus 5.5 --- roles/web-app-openproject/tasks/01_manager_ops.yml | 12 ++++++++++++ roles/web-app-openproject/templates/env.j2 | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/roles/web-app-openproject/tasks/01_manager_ops.yml b/roles/web-app-openproject/tasks/01_manager_ops.yml index bd0331f1d7..39bb5a9a80 100644 --- a/roles/web-app-openproject/tasks/01_manager_ops.yml +++ b/roles/web-app-openproject/tasks/01_manager_ops.yml @@ -29,6 +29,18 @@ - name: 🚿 flush docker service meta: flush_handlers +- name: ⏳ Wait until the seeder has committed the seed (swarm) + ansible.builtin.script: "{{ lookup('path_absolute', 'roles/sys-svc-compose/files/shell/swarm/job_state.sh') }}" + environment: + JOB: "{{ lookup('container_service', application_id, OPENPROJECT_SEEDER_SERVICE) }}" + register: openproject_seeder_state + changed_when: false + retries: 120 + delay: 10 + until: openproject_seeder_state.rc != 1 + failed_when: openproject_seeder_state.rc != 0 + when: DEPLOYMENT_MODE == 'swarm' + - name: 🚀 Deploy Settings include_tasks: 02_settings.yml diff --git a/roles/web-app-openproject/templates/env.j2 b/roles/web-app-openproject/templates/env.j2 index cc684cdccc..bbbb560daf 100644 --- a/roles/web-app-openproject/templates/env.j2 +++ b/roles/web-app-openproject/templates/env.j2 @@ -19,7 +19,7 @@ OPENPROJECT_AUTH__SOURCE__SSO_LOGOUT__URL={{ OPENPROJECT_SSO_LOGOUT_URL }} {% endif %} POSTGRES_PASSWORD={{ lookup('database', application_id, 'password') | dotenv_quote }} -DATABASE_URL={{ lookup('database', application_id, 'url_full') }}?pool=20&encoding=unicode&reconnect=true +DATABASE_URL={{ lookup('database', application_id, 'url_full') }}?pool=20&encoding=unicode&reconnect=true&keepalives=1&keepalives_idle=60&keepalives_interval=10&keepalives_count=6&tcp_user_timeout=120000 DB_POOL={{ OPENPROJECT_DB_POOL }} RAILS_MAX_THREADS={{ OPENPROJECT_RAILS_MAX_THREADS }} RAILS_MIN_THREADS=4