fix: Gitea Traefik routing and connection pool optimization

- Remove middleware reference from Gitea Traefik labels (caused routing issues) - Optimize Gitea connection pool settings (MAX_IDLE_CONNS=30, authentication_timeout=180s) - Add explicit service reference in Traefik labels - Fix intermittent 504 timeouts by improving PostgreSQL connection handling Fixes Gitea unreachability via git.michaelschiemer.de
2025-11-09 14:46:15 +01:00
parent 85c369e846
commit 36ef2a1e2c
1366 changed files with 104925 additions and 28719 deletions
--- a/deployment/ansible/roles/application/tasks/deploy.yml
+++ b/deployment/ansible/roles/application/tasks/deploy.yml
@@ -1,12 +1,326 @@
 ---
+- name: Debug all available variables before password determination
+  ansible.builtin.debug:
+    msg: |
+      Available variables for registry password:
+      - docker_registry_password_default defined: {{ docker_registry_password_default is defined }}
+      - vault_docker_registry_password defined: {{ vault_docker_registry_password is defined }}
+      - All vault_* variable names: {{ vars.keys() | select('match', '^vault_.*') | list | join(', ') }}
+  delegate_to: localhost
+  become: no
+
+- name: Check if docker_registry_password_default is set (safe check)
+  ansible.builtin.set_fact:
+    _docker_registry_password_default_set: "{{ 'YES' if (docker_registry_password_default is defined and docker_registry_password_default | string | trim != '') else 'NO' }}"
+  delegate_to: localhost
+  become: no
+  when: docker_registry_password_default is defined
+
+- name: Check if vault_docker_registry_password is set (safe check)
+  ansible.builtin.set_fact:
+    _vault_docker_registry_password_set: "{{ 'YES' if (vault_docker_registry_password is defined and vault_docker_registry_password | string | trim != '') else 'NO' }}"
+  delegate_to: localhost
+  become: no
+  when: vault_docker_registry_password is defined
+
+- name: Debug password status
+  ansible.builtin.debug:
+    msg: |
+      Password status:
+      - docker_registry_password_default: {{ _docker_registry_password_default_set | default('NOT DEFINED') }}
+      - vault_docker_registry_password: {{ _vault_docker_registry_password_set | default('NOT DEFINED') }}
+  delegate_to: localhost
+  become: no
+
+- name: Determine Docker registry password from vault or defaults
+  ansible.builtin.set_fact:
+    registry_password: >-
+      {%- if docker_registry_password_default is defined and docker_registry_password_default | string | trim != '' -%}
+        {{ docker_registry_password_default }}
+      {%- elif vault_docker_registry_password is defined and vault_docker_registry_password | string | trim != '' -%}
+        {{ vault_docker_registry_password }}
+      {%- else -%}
+        {{ '' }}
+      {%- endif -%}
+  no_log: yes
+
+- name: Debug registry password source after determination
+  ansible.builtin.debug:
+    msg: |
+      Registry password determination result:
+      - docker_registry_password_default: {{ 'SET (length: ' + (docker_registry_password_default | default('') | string | length | string) + ')' if (docker_registry_password_default | default('') | string | trim) != '' else 'NOT SET' }}
+      - vault_docker_registry_password defined: {{ vault_docker_registry_password is defined }}
+      - vault_docker_registry_password set: {{ 'YES (length: ' + (vault_docker_registry_password | default('') | string | length | string) + ')' if (vault_docker_registry_password | default('') | string | trim) != '' else 'NO' }}
+      - registry_password set: {{ 'YES (length: ' + (registry_password | default('') | string | length | string) + ')' if (registry_password | default('') | string | trim) != '' else 'NO' }}
+  delegate_to: localhost
+  become: no
+
+- name: Debug vault loading
+  ansible.builtin.debug:
+    msg: |
+      Vault loading status:
+      - Vault file exists: {{ application_vault_stat.stat.exists | default(false) }}
+      - vault_docker_registry_password defined: {{ vault_docker_registry_password is defined }}
+      - vault_docker_registry_password value: {{ 'SET (length: ' + (vault_docker_registry_password | default('') | string | length | string) + ')' if (vault_docker_registry_password | default('') | string | trim) != '' else 'NOT SET or EMPTY' }}
+      - registry_password: {{ 'SET (length: ' + (registry_password | default('') | string | length | string) + ')' if (registry_password | default('') | string | trim) != '' else 'NOT SET or EMPTY' }}
+  when: true
+  no_log: yes
+
+- name: Check if registry is accessible
+  ansible.builtin.uri:
+    url: "http://{{ docker_registry | default('localhost:5000') }}/v2/"
+    method: GET
+    status_code: [200, 401]
+    timeout: 5
+  register: registry_check
+  ignore_errors: yes
+  delegate_to: "{{ inventory_hostname }}"
+  become: no
+
+- name: Debug registry accessibility
+  ansible.builtin.debug:
+    msg: |
+      Registry accessibility check:
+      - Registry URL: http://{{ docker_registry | default('localhost:5000') }}/v2/
+      - Status code: {{ registry_check.status | default('UNKNOWN') }}
+      - Accessible: {{ 'YES' if registry_check.status | default(0) in [200, 401] else 'NO' }}
+      - Note: Status 401 means registry requires authentication (expected)
+  delegate_to: localhost
+  become: no
+
+- name: Login to Docker registry
+  community.docker.docker_login:
+    registry_url: "{{ docker_registry | default('localhost:5000') }}"
+    username: "{{ docker_registry_username_default | default('admin') }}"
+    password: "{{ registry_password }}"
+  when: 
+    - registry_password | string | trim != ''
+    - registry_check.status | default(0) in [200, 401]
+  no_log: yes
+  ignore_errors: yes
+  register: docker_login_result
+
+- name: Warn if Docker registry login failed
+  ansible.builtin.debug:
+    msg: "WARNING: Docker registry login failed or skipped. Images may not be pullable without authentication."
+  when: 
+    - registry_password | string | trim != ''
+    - docker_login_result.failed | default(false)
+
+- name: Debug registry authentication status
+  ansible.builtin.debug:
+    msg: |
+      Registry authentication status:
+      - Registry: {{ docker_registry | default('localhost:5000') }}
+      - Password set: {{ 'YES' if (registry_password | string | trim) != '' else 'NO' }}
+      - Login result: {{ 'SUCCESS' if (docker_login_result.failed | default(true) == false) else 'FAILED or SKIPPED' }}
+      - Username: {{ docker_registry_username_default | default('admin') }}
+  when: true
+
+- name: Fail if registry password is not set
+  ansible.builtin.fail:
+    msg: |
+      Docker registry authentication required but password not set!
+      
+      The registry at {{ docker_registry | default('localhost:5000') }} requires authentication.
+      
+      Please set the password in one of these ways:
+      
+      1. Set in vault file (recommended):
+         ansible-vault edit {{ vault_file | default('inventory/group_vars/production/vault.yml') }}
+         # Add: vault_docker_registry_password: "your-password"
+      
+      2. Pass via extra vars:
+         -e "docker_registry_password_default=your-password"
+      
+      3. Use init-secrets.sh script to generate all passwords:
+         cd deployment/ansible
+         ./scripts/init-secrets.sh
+      
+      Note: The registry password was likely generated when the registry stack was deployed.
+      Check the registry role output or the vault file for the generated password.
+  when:
+    - registry_password | string | trim == ''
+    - docker_registry | default('localhost:5000') == 'localhost:5000'
+
+- name: Check registry htpasswd file to verify password
+  ansible.builtin.shell: |
+    if [ -f "{{ registry_auth_path | default('/home/deploy/deployment/stacks/registry/auth') }}/htpasswd" ]; then
+      cat "{{ registry_auth_path | default('/home/deploy/deployment/stacks/registry/auth') }}/htpasswd"
+    else
+      echo "htpasswd file not found"
+    fi
+  register: registry_htpasswd_check
+  changed_when: false
+  failed_when: false
+  delegate_to: "{{ inventory_hostname }}"
+  become: no
+  when: docker_login_result.failed | default(false)
+
+- name: Debug registry password mismatch
+  ansible.builtin.debug:
+    msg: |
+      Registry authentication failed!
+      
+      Registry: {{ docker_registry | default('localhost:5000') }}
+      Username: {{ docker_registry_username_default | default('admin') }}
+      
+      Possible causes:
+      1. The password in vault does not match the password used during registry deployment
+      2. The registry was deployed with a different password (generated by registry role)
+      3. The username is incorrect
+      
+      To fix:
+      1. Check the registry htpasswd file on the server:
+         cat {{ registry_auth_path | default('/home/deploy/deployment/stacks/registry/auth') }}/htpasswd
+      
+      2. Extract the password from the registry .env file (if available):
+         grep REGISTRY_AUTH {{ registry_stack_path | default('/home/deploy/deployment/stacks/registry') }}/.env
+      
+      3. Update the vault file with the correct password:
+         ansible-vault edit {{ vault_file | default('inventory/group_vars/production/vault.yml') }}
+         # Set: vault_docker_registry_password: "correct-password"
+      
+      4. Or re-deploy the registry stack with the password from vault:
+         ansible-playbook -i inventory/production.yml playbooks/setup-infrastructure.yml --tags registry
+      
+      Registry htpasswd file content:
+      {{ registry_htpasswd_check.stdout | default('NOT FOUND') }}
+  when:
+    - registry_password | string | trim != ''
+    - docker_login_result.failed | default(false)
+
+- name: Fail if registry authentication failed and password was provided
+  ansible.builtin.fail:
+    msg: |
+      Docker registry authentication failed!
+      
+      Registry: {{ docker_registry | default('localhost:5000') }}
+      Username: {{ docker_registry_username_default | default('admin') }}
+      
+      The password in the vault file does not match the password used during registry deployment.
+      Please check the debug output above for instructions on how to fix this.
+  when:
+    - registry_password | string | trim != ''
+    - docker_login_result.failed | default(false)
+
+- name: Force pull latest Docker images before deployment
+  shell: |
+    docker compose -f {{ application_stack_dest }}/docker-compose.base.yml -f {{ application_stack_dest }}/docker-compose.{{ application_compose_suffix }} pull --ignore-pull-failures
+  changed_when: false
+  failed_when: false
+  when: not ansible_check_mode
+
+- name: Verify entrypoint script exists in Docker image (method 1 - file check)
+  shell: |
+    docker run --rm --entrypoint=/bin/sh {{ docker_registry | default('localhost:5000') }}/{{ app_name | default('framework') }}:latest -c "test -f /usr/local/bin/entrypoint.sh && ls -la /usr/local/bin/entrypoint.sh || echo 'FILE_NOT_FOUND'"
+  register: entrypoint_check
+  changed_when: false
+  failed_when: false
+
+- name: Verify entrypoint script exists in Docker image (method 2 - inspect image)
+  shell: |
+    docker image inspect {{ docker_registry | default('localhost:5000') }}/{{ app_name | default('framework') }}:latest --format '{{ "{{" }}.Config.Entrypoint{{ "}}" }}' 2>&1 || echo "INSPECT_FAILED"
+  register: entrypoint_inspect
+  changed_when: false
+  failed_when: false
+
+- name: Verify entrypoint script exists in Docker image (method 3 - extract and check)
+  shell: |
+    CONTAINER_ID=$(docker create {{ docker_registry | default('localhost:5000') }}/{{ app_name | default('framework') }}:latest 2>/dev/null) && \
+    docker cp $CONTAINER_ID:/usr/local/bin/entrypoint.sh /tmp/entrypoint_check.sh 2>&1 && \
+    if [ -f /tmp/entrypoint_check.sh ]; then \
+      echo "FILE_EXISTS"; \
+      ls -la /tmp/entrypoint_check.sh; \
+      head -5 /tmp/entrypoint_check.sh; \
+      rm -f /tmp/entrypoint_check.sh; \
+    else \
+      echo "FILE_NOT_FOUND"; \
+    fi && \
+    docker rm $CONTAINER_ID >/dev/null 2>&1 || true
+  register: entrypoint_extract
+  changed_when: false
+  failed_when: false
+
+- name: Set entrypoint verification message
+  set_fact:
+    entrypoint_verification_msg: |
+      ==========================================
+      Entrypoint Script Verification
+      ==========================================
+      Image: {{ docker_registry | default('localhost:5000') }}/{{ app_name | default('framework') }}:latest
+      
+      Method 1 - File Check:
+      Return Code: {{ entrypoint_check.rc | default('unknown') }}
+      Output: {{ entrypoint_check.stdout | default('No output') }}
+      
+      Method 2 - Image Inspect:
+      Entrypoint Config: {{ entrypoint_inspect.stdout | default('Not available') }}
+      
+      Method 3 - Extract and Check:
+      {{ entrypoint_extract.stdout | default('Check not performed') }}
+      
+      {% if 'FILE_NOT_FOUND' in entrypoint_check.stdout or 'FILE_NOT_FOUND' in entrypoint_extract.stdout %}
+      ⚠️  WARNING: Entrypoint script NOT FOUND in image!
+      
+      This means the Docker image was built without the entrypoint script.
+      Possible causes:
+      1. The entrypoint script was not copied during rsync to build directory
+      2. The Dockerfile COPY command failed silently
+      3. The image needs to be rebuilt with --no-cache
+      
+      Next steps:
+      1. Rebuild the image: ansible-playbook -i inventory/production.yml playbooks/build-initial-image.yml --vault-password-file secrets/.vault_pass -e "build_no_cache=true"
+      2. Check if docker/entrypoint.sh exists on server: ls -la /home/deploy/michaelschiemer/docker/entrypoint.sh
+      3. Manually check image: docker run --rm --entrypoint=/bin/sh localhost:5000/framework:latest -c "ls -la /usr/local/bin/entrypoint.sh"
+      {% elif entrypoint_check.rc == 0 %}
+      ✅ Entrypoint script found in image
+      File details: {{ entrypoint_check.stdout }}
+      {% if '\r' in entrypoint_extract.stdout %}
+      ⚠️  CRITICAL: Entrypoint script has CRLF line endings!
+      The script contains \r characters which will cause "no such file or directory" errors.
+      The script needs to be converted to LF line endings before building the image.
+      {% endif %}
+      {% else %}
+      ⚠️  Could not verify entrypoint script (check may have failed)
+      {% endif %}
+      ==========================================
+
+- name: Display entrypoint script verification result
+  debug:
+    var: entrypoint_verification_msg
+
 - name: Deploy application stack
  community.docker.docker_compose_v2:
    project_src: "{{ application_stack_dest }}"
+    files:
+      - docker-compose.base.yml
+      - "docker-compose.{{ application_compose_suffix }}"
    state: present
    pull: always
    recreate: "{{ application_compose_recreate }}"
    remove_orphans: "{{ application_remove_orphans | bool }}"
  register: application_compose_result
+  failed_when: false
+
+- name: Show PHP container logs if deployment failed
+  shell: |
+    docker compose -f {{ application_stack_dest }}/docker-compose.base.yml -f {{ application_stack_dest }}/docker-compose.{{ application_compose_suffix }} logs --tail=50 {{ application_service_name }} 2>&1 || true
+  register: application_php_logs
+  changed_when: false
+  when: application_compose_result.failed | default(false)
+
+- name: Display PHP container logs on failure
+  debug:
+    msg: |
+      PHP Container Logs (last 50 lines):
+      {{ application_php_logs.stdout | default('No logs available') }}
+  when: application_compose_result.failed | default(false)
+
+- name: Fail if deployment failed
+  fail:
+    msg: "Application stack deployment failed. Check logs above for details."
+  when: application_compose_result.failed | default(false)

 - name: Wait for application container to report Up
  shell: |
@@ -17,6 +331,48 @@
  retries: "{{ ((application_wait_timeout | int) + (application_wait_interval | int) - 1) // (application_wait_interval | int) }}"
  delay: "{{ application_wait_interval | int }}"
  when: application_compose_result.changed
+  failed_when: false
+
+- name: Show container status when container doesn't start
+  shell: |
+    docker compose -f {{ application_stack_dest }}/docker-compose.base.yml -f {{ application_stack_dest }}/docker-compose.{{ application_compose_suffix }} ps {{ application_service_name }}
+  register: application_container_status
+  changed_when: false
+  when: 
+    - application_compose_result.changed
+    - application_app_running.rc != 0
+
+- name: Show PHP container logs when container doesn't start
+  shell: |
+    docker compose -f {{ application_stack_dest }}/docker-compose.base.yml -f {{ application_stack_dest }}/docker-compose.{{ application_compose_suffix }} logs --tail=100 {{ application_service_name }} 2>&1 || true
+  register: application_php_logs_failed
+  changed_when: false
+  when: 
+    - application_compose_result.changed
+    - application_app_running.rc != 0
+
+- name: Display container status and logs when startup failed
+  debug:
+    msg: |
+      Container Status:
+      {{ application_container_status.stdout | default('Container not found') }}
+      
+      Container Logs (last 100 lines):
+      {{ application_php_logs_failed.stdout | default('No logs available') }}
+  when: 
+    - application_compose_result.changed
+    - application_app_running.rc != 0
+
+- name: Fail if container didn't start
+  fail:
+    msg: |
+      Application container '{{ application_service_name }}' failed to start.
+      Check the logs above for details.
+      You can also check manually with:
+      docker compose -f {{ application_stack_dest }}/docker-compose.base.yml -f {{ application_stack_dest }}/docker-compose.{{ application_compose_suffix }} logs {{ application_service_name }}
+  when: 
+    - application_compose_result.changed
+    - application_app_running.rc != 0

 - name: Ensure app container is running before migrations
  shell: |