From 7a2cb0b63e8ef82d8aa313cba4beac306ff66692 Mon Sep 17 00:00:00 2001 From: Michael Schiemer Date: Mon, 3 Nov 2025 00:15:43 +0100 Subject: [PATCH] fix: staging deployment configuration and redis secrets handling --- .gitea/workflows/build-image.yml | 6 +- Makefile | 14 +- .../playbooks/debug-staging-redis-secrets.yml | 250 ++++++++++++ .../ansible/playbooks/deploy/production.yml | 239 ++++++++++++ .../ansible/playbooks/deploy/staging.yml | 226 +++++++++++ .../playbooks/fix-staging-compose-secrets.yml | 138 +++++++ .../fix-staging-secrets-permissions.yml | 82 ++++ .../semaphore/playbooks/deploy-staging.yml | 5 +- deployment/stacks/staging/README.md | 31 +- deployment/stacks/staging/docker-compose.yml | 356 ------------------ docker-compose.staging.yml | 66 +++- src/Framework/Redis/RedisConnection.php | 2 +- 12 files changed, 1027 insertions(+), 388 deletions(-) create mode 100644 deployment/ansible/playbooks/debug-staging-redis-secrets.yml create mode 100644 deployment/ansible/playbooks/deploy/production.yml create mode 100644 deployment/ansible/playbooks/deploy/staging.yml create mode 100644 deployment/ansible/playbooks/fix-staging-compose-secrets.yml create mode 100644 deployment/ansible/playbooks/fix-staging-secrets-permissions.yml delete mode 100644 deployment/stacks/staging/docker-compose.yml diff --git a/.gitea/workflows/build-image.yml b/.gitea/workflows/build-image.yml index 6d51e6fe..9da5efd7 100644 --- a/.gitea/workflows/build-image.yml +++ b/.gitea/workflows/build-image.yml @@ -826,11 +826,9 @@ jobs: # Job 3: Auto-deploy to Staging (only for staging branch) deploy-staging: name: Auto-deploy to Staging - needs: [changes, build, runtime-base] + needs: [changes, build] if: | - (github.ref_name == 'staging' || github.head_ref == 'staging' || (github.ref_name == '' && contains(github.ref, 'staging'))) && - (needs.build.result == 'success' || needs.build.result == 'skipped') && - (needs.runtime-base.result == 'success' || needs.runtime-base.result == 'skipped') + github.ref_name == 'staging' || github.head_ref == 'staging' || (github.ref_name == '' && contains(github.ref, 'staging')) runs-on: ubuntu-latest environment: name: staging diff --git a/Makefile b/Makefile index d10f3599..8d179c01 100644 --- a/Makefile +++ b/Makefile @@ -218,27 +218,27 @@ health: # Ansible Konfiguration -ANSIBLE_INVENTORY=ansible/inventory/hosts.ini -PLAYBOOK_DIR=ansible/playbooks/deploy +ANSIBLE_INVENTORY=deployment/ansible/inventory/production.yml +PLAYBOOK_DIR=deployment/ansible/playbooks/deploy TAGS= .PHONY: dev staging production setup-server check # Deployment-Ziele dev: ## Lokales Deployment (Development) - ansible-playbook -i $(ANSIBLE_INVENTORY) $(PLAYBOOK_DIR)/dev.yml --ask-become-pass $(if $(TAGS),--tags="$(TAGS)",) + cd deployment/ansible && ansible-playbook -i inventory/production.yml playbooks/deploy/dev.yml --ask-become-pass $(if $(TAGS),--tags="$(TAGS)",) staging: ## Staging-Deployment - ansible-playbook -i $(ANSIBLE_INVENTORY) $(PLAYBOOK_DIR)/staging.yml $(if $(TAGS),--tags="$(TAGS)",) + cd deployment/ansible && ansible-playbook -i inventory/production.yml playbooks/deploy/staging.yml $(if $(TAGS),--tags="$(TAGS)",) production: ## Produktions-Deployment - ansible-playbook -i $(ANSIBLE_INVENTORY) $(PLAYBOOK_DIR)/production.yml $(if $(TAGS),--tags="$(TAGS)",) + cd deployment/ansible && ansible-playbook -i inventory/production.yml playbooks/deploy/production.yml $(if $(TAGS),--tags="$(TAGS)",) setup-server: ## Server-Grundkonfiguration - ansible-playbook -i $(ANSIBLE_INVENTORY) ansible/setup.yml $(if $(LIMIT),--limit="$(LIMIT)",) $(if $(TAGS),--tags="$(TAGS)",) + cd deployment/ansible && ansible-playbook -i inventory/production.yml playbooks/setup-infrastructure.yml $(if $(LIMIT),--limit="$(LIMIT)",) $(if $(TAGS),--tags="$(TAGS)",) check: ## Serververbindung prΓΌfen - ansible -i $(ANSIBLE_INVENTORY) all -m ping $(if $(LIMIT),--limit="$(LIMIT)",) + cd deployment/ansible && ansible -i inventory/production.yml all -m ping $(if $(LIMIT),--limit="$(LIMIT)",) # Beispielaufrufe: # make staging TAGS="deploy,check" diff --git a/deployment/ansible/playbooks/debug-staging-redis-secrets.yml b/deployment/ansible/playbooks/debug-staging-redis-secrets.yml new file mode 100644 index 00000000..6d7e0a42 --- /dev/null +++ b/deployment/ansible/playbooks/debug-staging-redis-secrets.yml @@ -0,0 +1,250 @@ +--- +- name: Debug Staging Redis Secrets Configuration + hosts: production + gather_facts: yes + become: no + + tasks: + - name: Check staging stack directory + shell: | + cd ~/deployment/stacks/staging + echo "=== Staging Stack Directory ===" + pwd + ls -la + register: dir_check + ignore_errors: yes + + - name: Display directory contents + debug: + msg: "{{ dir_check.stdout_lines }}" + + - name: Check if docker-compose files exist + stat: + path: "{{ item }}" + vars: + deployment_path: "~/deployment/stacks/staging" + with_items: + - "{{ deployment_path }}/docker-compose.base.yml" + - "{{ deployment_path }}/docker-compose.staging.yml" + register: compose_files + + - name: Display compose file status + debug: + msg: "{{ item.stat.exists | default(false) }}: {{ item.item }}" + with_items: "{{ compose_files.results }}" + + - name: Check docker-compose.staging.yml configuration + shell: | + cd ~/deployment/stacks/staging + echo "=== Checking docker-compose.staging.yml for Redis secrets ===" + if [ -f docker-compose.staging.yml ]; then + echo "--- REDIS_PASSWORD_FILE in environment ---" + grep -A 5 "staging-app:" docker-compose.staging.yml | grep -A 10 "environment:" | grep "REDIS_PASSWORD_FILE" || echo "REDIS_PASSWORD_FILE not found in staging-app environment" + echo "" + echo "--- Secrets section for staging-app ---" + grep -A 10 "staging-app:" docker-compose.staging.yml | grep -A 15 "secrets:" | head -10 || echo "Secrets section not found" + echo "" + echo "--- Secrets definitions at bottom ---" + tail -30 docker-compose.staging.yml | grep -A 5 "redis_password:" || echo "redis_password secret definition not found" + else + echo "docker-compose.staging.yml NOT FOUND" + fi + register: compose_config + ignore_errors: yes + + - name: Display compose configuration + debug: + msg: "{{ compose_config.stdout_lines }}" + + - name: Check if secrets directory and files exist + shell: | + cd ~/deployment/stacks/staging + echo "=== Secrets Directory ===" + if [ -d secrets ]; then + echo "secrets/ directory exists" + ls -la secrets/ + echo "" + echo "--- redis_password.txt content check ---" + if [ -f secrets/redis_password.txt ]; then + echo "secrets/redis_password.txt exists" + FILE_SIZE=$(stat -f%z secrets/redis_password.txt 2>/dev/null || stat -c%s secrets/redis_password.txt 2>/dev/null || echo "unknown") + CONTENT_LENGTH=$(wc -c < secrets/redis_password.txt | tr -d ' ') + echo "File size: $FILE_SIZE bytes" + echo "Content length: $CONTENT_LENGTH bytes" + # Show first 5 chars (for debugging) + FIRST_CHARS=$(head -c 5 secrets/redis_password.txt) + echo "First 5 chars: $FIRST_CHARS" + else + echo "secrets/redis_password.txt NOT FOUND" + fi + else + echo "secrets/ directory DOES NOT EXIST" + fi + register: secrets_check + ignore_errors: yes + + - name: Display secrets check + debug: + msg: "{{ secrets_check.stdout_lines }}" + + - name: Check if staging-app container is running + shell: | + docker ps --filter "name=staging-app" --format "{{.Names}}\t{{.Status}}\t{{.Image}}" + register: container_status + ignore_errors: yes + + - name: Display container status + debug: + msg: "{{ container_status.stdout_lines }}" + + - name: Check Docker secrets mounted in staging-app container + shell: | + echo "=== Docker Secrets in staging-app Container ===" + if docker ps --filter "name=staging-app" --format "{{.Names}}" | grep -q staging-app; then + echo "--- Checking /run/secrets/ directory ---" + docker exec staging-app ls -la /run/secrets/ 2>&1 || echo "Cannot access /run/secrets/" + echo "" + echo "--- Checking redis_password secret file ---" + docker exec staging-app cat /run/secrets/redis_password 2>&1 | head -c 20 || echo "redis_password secret NOT FOUND or NOT READABLE" + echo "..." + echo "" + echo "--- File exists check ---" + docker exec staging-app test -f /run/secrets/redis_password && echo "redis_password file EXISTS" || echo "redis_password file DOES NOT EXIST" + docker exec staging-app test -r /run/secrets/redis_password && echo "redis_password file is READABLE" || echo "redis_password file is NOT READABLE" + else + echo "staging-app container is NOT RUNNING" + fi + register: secrets_mounted + ignore_errors: yes + + - name: Display secrets mount status + debug: + msg: "{{ secrets_mounted.stdout_lines }}" + + - name: Check Environment Variables in staging-app container + shell: | + echo "=== Environment Variables in staging-app ===" + if docker ps --filter "name=staging-app" --format "{{.Names}}" | grep -q staging-app; then + echo "--- Redis-related environment variables ---" + docker exec staging-app env | grep -E "(REDIS_|CACHE_|SESSION_|QUEUE_)" || echo "No Redis env vars found" + echo "" + echo "--- *_FILE environment variables ---" + docker exec staging-app env | grep "_FILE" || echo "No _FILE env vars found" + echo "" + echo "--- All environment variables (first 50) ---" + docker exec staging-app env | sort | head -50 || echo "Cannot read environment" + else + echo "Container not running" + fi + register: env_vars + ignore_errors: yes + + - name: Display environment variables + debug: + msg: "{{ env_vars.stdout_lines }}" + + - name: Test PHP environment resolution (check DockerSecretsResolver) + shell: | + echo "=== Testing PHP Environment Resolution ===" + docker exec staging-app php -r " + // Simulate what the Framework does + echo '=== System Environment Check ===' . PHP_EOL; + echo 'getenv(\"REDIS_PASSWORD_FILE\"): ' . (getenv('REDIS_PASSWORD_FILE') ?: 'NOT SET') . PHP_EOL; + echo 'getenv(\"REDIS_PASSWORD\"): ' . (getenv('REDIS_PASSWORD') ? 'SET (length: ' . strlen(getenv('REDIS_PASSWORD')) . ')' : 'NOT SET') . PHP_EOL; + echo PHP_EOL; + + echo '=== $_ENV Check ===' . PHP_EOL; + echo 'isset($_ENV[\"REDIS_PASSWORD_FILE\"]): ' . (isset(\$_ENV['REDIS_PASSWORD_FILE']) ? 'YES: ' . \$_ENV['REDIS_PASSWORD_FILE'] : 'NO') . PHP_EOL; + echo 'isset($_ENV[\"REDIS_PASSWORD\"]): ' . (isset(\$_ENV['REDIS_PASSWORD']) ? 'YES (length: ' . strlen(\$_ENV['REDIS_PASSWORD']) . ')' : 'NO') . PHP_EOL; + echo PHP_EOL; + + echo '=== $_SERVER Check ===' . PHP_EOL; + echo 'isset($_SERVER[\"REDIS_PASSWORD_FILE\"]): ' . (isset(\$_SERVER['REDIS_PASSWORD_FILE']) ? 'YES: ' . \$_SERVER['REDIS_PASSWORD_FILE'] : 'NO') . PHP_EOL; + echo 'isset($_SERVER[\"REDIS_PASSWORD\"]): ' . (isset(\$_SERVER['REDIS_PASSWORD']) ? 'YES (length: ' . strlen(\$_SERVER['REDIS_PASSWORD']) . ')' : 'NO') . PHP_EOL; + echo PHP_EOL; + + echo '=== Docker Secrets File Check ===' . PHP_EOL; + \$secret_file = '/run/secrets/redis_password'; + echo 'File path: ' . \$secret_file . PHP_EOL; + echo 'File exists: ' . (file_exists(\$secret_file) ? 'YES' : 'NO') . PHP_EOL; + if (file_exists(\$secret_file)) { + echo 'File readable: ' . (is_readable(\$secret_file) ? 'YES' : 'NO') . PHP_EOL; + \$content = file_get_contents(\$secret_file); + if (\$content !== false) { + echo 'File content length: ' . strlen(trim(\$content)) . PHP_EOL; + echo 'File content (first 10 chars): ' . substr(trim(\$content), 0, 10) . '...' . PHP_EOL; + } else { + echo 'File content: COULD NOT READ' . PHP_EOL; + } + } + echo PHP_EOL; + + // Test DockerSecretsResolver logic + echo '=== DockerSecretsResolver Simulation ===' . PHP_EOL; + \$variables = getenv(); + \$file_key = 'REDIS_PASSWORD_FILE'; + if (isset(\$variables[\$file_key])) { + \$file_path = \$variables[\$file_key]; + echo 'REDIS_PASSWORD_FILE found: ' . \$file_path . PHP_EOL; + if (file_exists(\$file_path) && is_readable(\$file_path)) { + \$secret_value = trim(file_get_contents(\$file_path)); + echo 'Secret resolved: YES (length: ' . strlen(\$secret_value) . ')' . PHP_EOL; + echo 'Secret value (first 10 chars): ' . substr(\$secret_value, 0, 10) . '...' . PHP_EOL; + } else { + echo 'Secret resolved: NO (file not accessible)' . PHP_EOL; + } + } else { + echo 'REDIS_PASSWORD_FILE NOT FOUND in environment' . PHP_EOL; + } + " 2>&1 + register: php_test + ignore_errors: yes + + - name: Display PHP environment test + debug: + msg: "{{ php_test.stdout_lines }}" + + - name: Check staging-redis container configuration + shell: | + echo "=== Staging Redis Container ===" + docker ps --filter "name=staging-redis" --format "{{.Names}}\t{{.Status}}" + echo "" + echo "=== Redis password requirement ===" + docker exec staging-redis redis-cli CONFIG GET requirepass 2>&1 || echo "Cannot check Redis config" + echo "" + echo "=== Test Redis connection without password ===" + docker exec staging-redis redis-cli PING 2>&1 || echo "Connection failed (password required)" + register: redis_config + ignore_errors: yes + + - name: Display Redis configuration + debug: + msg: "{{ redis_config.stdout_lines }}" + + - name: Check recent staging-app logs for Redis errors + shell: | + cd ~/deployment/stacks/staging + echo "=== Recent staging-app logs (Redis-related) ===" + docker compose -f docker-compose.base.yml -f docker-compose.staging.yml logs staging-app --tail=100 2>&1 | grep -i -E "(redis|password|secret|auth|noauth)" | tail -30 || echo "No Redis-related logs found" + register: app_logs + ignore_errors: yes + + - name: Display application logs + debug: + msg: "{{ app_logs.stdout_lines }}" + + - name: Summary and Recommendations + debug: + msg: + - "========================================" + - "DEBUG SUMMARY" + - "========================================" + - "Check the output above for:" + - "1. docker-compose.staging.yml has REDIS_PASSWORD_FILE=/run/secrets/redis_password" + - "2. secrets/redis_password.txt exists and is readable" + - "3. Container has /run/secrets/redis_password file mounted" + - "4. Container environment has REDIS_PASSWORD_FILE variable set" + - "5. PHP can read the secret file and resolve REDIS_PASSWORD" + - "6. Redis container requires password (requirepass set)" + - "" + - "If any check fails, the issue is identified above." diff --git a/deployment/ansible/playbooks/deploy/production.yml b/deployment/ansible/playbooks/deploy/production.yml new file mode 100644 index 00000000..beca23e5 --- /dev/null +++ b/deployment/ansible/playbooks/deploy/production.yml @@ -0,0 +1,239 @@ +--- +- name: Deploy Application Update to Production via Docker Compose + hosts: production + gather_facts: yes + become: no + + vars: + # These should be passed via -e from CI/CD + application_environment: production + application_compose_suffix: production.yml + # app_stack_path is now defined in group_vars/production.yml + + pre_tasks: + - name: Set deployment variables + set_fact: + image_tag: "{{ image_tag | default('latest') }}" + git_commit_sha: "{{ git_commit_sha | default('unknown') }}" + deployment_timestamp: "{{ deployment_timestamp | default(ansible_date_time.iso8601) }}" + - name: Optionally load registry credentials from encrypted vault + include_vars: + file: "{{ playbook_dir }}/../../secrets/production.vault.yml" + no_log: yes + ignore_errors: yes + delegate_to: localhost + become: no + + - name: Derive docker registry credentials from vault when not provided + set_fact: + docker_registry_username: "{{ docker_registry_username | default(vault_docker_registry_username | default(docker_registry_username_default)) }}" + docker_registry_password: "{{ docker_registry_password | default(vault_docker_registry_password | default(docker_registry_password_default)) }}" + + - name: Ensure system packages are up to date + include_role: + name: system + when: system_update_packages | bool + + - name: Verify Docker is running + systemd: + name: docker + state: started + register: docker_service + become: yes + + - name: Fail if Docker is not running + fail: + msg: "Docker service is not running" + when: docker_service.status.ActiveState != 'active' + + - name: Ensure application stack directory exists + file: + path: "{{ app_stack_path }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + + - name: Check if docker-compose.base.yml exists in application stack + stat: + path: "{{ app_stack_path }}/docker-compose.base.yml" + register: compose_base_exists + when: not (application_sync_files | default(false) | bool) + + - name: Check if docker-compose.production.yml exists in application stack + stat: + path: "{{ app_stack_path }}/docker-compose.production.yml" + register: compose_override_exists + when: not (application_sync_files | default(false) | bool) + + - name: Fail if docker-compose files don't exist + fail: + msg: | + Application Stack docker-compose files not found at {{ app_stack_path }} + + Required files: + - docker-compose.base.yml + - docker-compose.production.yml + + The Application Stack must be deployed first via: + ansible-playbook -i inventory/production.yml playbooks/setup-infrastructure.yml + + This will create the application stack with docker-compose files and .env file. + when: + - not (application_sync_files | default(false) | bool) + - (not compose_base_exists.stat.exists or not compose_override_exists.stat.exists) + + - name: Create backup directory + file: + path: "{{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + + tasks: + - name: Verify docker-compose files exist + stat: + path: "{{ app_stack_path }}/docker-compose.base.yml" + register: compose_base_check + when: not (application_sync_files | default(false) | bool) + + - name: Verify docker-compose.production.yml exists + stat: + path: "{{ app_stack_path }}/docker-compose.production.yml" + register: compose_override_check + when: not (application_sync_files | default(false) | bool) + + - name: Fail if docker-compose files don't exist + fail: + msg: | + Application Stack docker-compose files not found at {{ app_stack_path }} + + Required files: + - docker-compose.base.yml + - docker-compose.production.yml + + The Application Stack must be deployed first via: + ansible-playbook -i inventory/production.yml playbooks/setup-infrastructure.yml + + This will create the application stack with docker-compose files and .env file. + when: + - not (application_sync_files | default(false) | bool) + - (not compose_base_check.stat.exists or not compose_override_check.stat.exists) + + - name: Backup current deployment metadata + shell: | + docker compose -f {{ app_stack_path }}/docker-compose.base.yml -f {{ app_stack_path }}/docker-compose.production.yml ps --format json 2>/dev/null > {{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}/current_containers.json || true + docker compose -f {{ app_stack_path }}/docker-compose.base.yml -f {{ app_stack_path }}/docker-compose.production.yml config 2>/dev/null > {{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}/docker-compose-config.yml || true + args: + executable: /bin/bash + changed_when: false + ignore_errors: yes + when: + - not (application_sync_files | default(false) | bool) + - compose_base_exists.stat.exists | default(false) + - compose_override_exists.stat.exists | default(false) + + - name: Login to Docker registry (if credentials provided) + community.docker.docker_login: + registry_url: "{{ docker_registry_url }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + no_log: yes + ignore_errors: yes + when: + - docker_registry_username is defined + - docker_registry_password is defined + - docker_registry_username | length > 0 + - docker_registry_password | length > 0 + register: registry_login + + - name: Pull new Docker image + community.docker.docker_image: + name: "{{ app_image }}" + tag: "{{ image_tag }}" + source: pull + force_source: yes + register: image_pull + + - name: Verify image was pulled successfully + fail: + msg: "Failed to pull image {{ app_image }}:{{ image_tag }}" + when: image_pull.failed + + # Sync files first if application_sync_files=true (before updating docker-compose.production.yml) + - name: Sync application stack files + import_role: + name: application + vars: + application_sync_files: "{{ application_sync_files | default(false) }}" + application_compose_recreate: "never" # Don't recreate yet, just sync files + application_remove_orphans: false + when: application_sync_files | default(false) | bool + + - name: Update docker-compose.production.yml with new image tag (all services) + replace: + path: "{{ app_stack_path }}/docker-compose.production.yml" + # Match both localhost:5000 and registry.michaelschiemer.de (or any registry URL) + regexp: '^(\s+image:\s+)(localhost:5000|registry\.michaelschiemer\.de|{{ docker_registry }})/{{ app_name }}:.*$' + replace: '\1{{ app_image }}:{{ image_tag }}' + # Always update to ensure localhost:5000 is used (registry only accessible via localhost) + when: true + register: compose_updated + + - name: Redeploy application stack with new image + import_role: + name: application + vars: + application_sync_files: false # Already synced above, don't sync again + application_compose_recreate: "always" + application_remove_orphans: true + + - name: Get deployed image information + shell: | + docker compose -f {{ app_stack_path }}/docker-compose.base.yml -f {{ app_stack_path }}/docker-compose.production.yml config | grep -E "^\s+image:" | head -1 | awk '{print $2}' || echo "unknown" + args: + executable: /bin/bash + register: deployed_image + changed_when: false + + - name: Record deployment metadata + copy: + content: | + Deployment Timestamp: {{ deployment_timestamp }} + Git Commit: {{ git_commit_sha }} + Image Tag: {{ image_tag }} + Deployed Image: {{ deployed_image.stdout }} + Image Pull: {{ 'SUCCESS' if image_pull.changed else 'SKIPPED (already exists)' }} + Stack Deploy: {{ 'UPDATED' if application_stack_changed else 'NO_CHANGE' }} + Health Status: {{ application_health_output if application_health_output != '' else 'All services healthy' }} + Health Check HTTP Status: {{ application_healthcheck_status }} + dest: "{{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}/deployment_metadata.txt" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' + + - name: Cleanup old backups (keep last {{ max_rollback_versions | default(5) }}) + shell: | + cd {{ backups_path }} + ls -dt */ 2>/dev/null | tail -n +{{ max_rollback_versions | default(5) + 1 }} | xargs -r rm -rf + args: + executable: /bin/bash + changed_when: false + ignore_errors: yes + + post_tasks: + - name: Display deployment summary + debug: + msg: + - "=== Production Deployment Summary ===" + - "Image: {{ app_image }}:{{ image_tag }}" + - "Commit: {{ git_commit_sha }}" + - "Timestamp: {{ deployment_timestamp }}" + - "Image Pull: {{ 'SUCCESS' if image_pull.changed else 'SKIPPED' }}" + - "Stack Deploy: {{ 'UPDATED' if application_stack_changed else 'NO_CHANGE' }}" + - "Health Output: {{ application_health_output if application_health_output != '' else 'All services healthy' }}" + - "Health Check HTTP Status: {{ application_healthcheck_status }}" + - "Health Check URL: {{ health_check_url | default('https://michaelschiemer.de/health') }}" + - "" + - "Next: Verify application is healthy" diff --git a/deployment/ansible/playbooks/deploy/staging.yml b/deployment/ansible/playbooks/deploy/staging.yml new file mode 100644 index 00000000..c15be4a8 --- /dev/null +++ b/deployment/ansible/playbooks/deploy/staging.yml @@ -0,0 +1,226 @@ +--- +- name: Deploy Application Update to Staging via Docker Compose + hosts: production + gather_facts: yes + become: no + + vars: + # These should be passed via -e from CI/CD + application_environment: staging + application_compose_suffix: staging.yml + # app_stack_path is now defined in group_vars/production.yml + + pre_tasks: + - name: Set deployment variables + set_fact: + image_tag: "{{ image_tag | default('latest') }}" + git_commit_sha: "{{ git_commit_sha | default('unknown') }}" + deployment_timestamp: "{{ deployment_timestamp | default(ansible_date_time.iso8601) }}" + - name: Optionally load registry credentials from encrypted vault + include_vars: + file: "{{ playbook_dir }}/../../secrets/production.vault.yml" + no_log: yes + ignore_errors: yes + delegate_to: localhost + become: no + + - name: Derive docker registry credentials from vault when not provided + set_fact: + docker_registry_username: "{{ docker_registry_username | default(vault_docker_registry_username | default(docker_registry_username_default)) }}" + docker_registry_password: "{{ docker_registry_password | default(vault_docker_registry_password | default(docker_registry_password_default)) }}" + + - name: Ensure system packages are up to date + include_role: + name: system + when: system_update_packages | bool + + - name: Verify Docker is running + systemd: + name: docker + state: started + register: docker_service + become: yes + + - name: Fail if Docker is not running + fail: + msg: "Docker service is not running" + when: docker_service.status.ActiveState != 'active' + + - name: Set staging stack path + set_fact: + app_stack_path: "{{ staging_stack_path | default(stacks_base_path + '/staging') }}" + backups_path: "{{ backups_base_path | default('~/deployment/backups') }}" + + - name: Ensure application stack directory exists + file: + path: "{{ app_stack_path }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + + - name: Check if docker-compose.base.yml exists in staging stack + stat: + path: "{{ app_stack_path }}/docker-compose.base.yml" + register: compose_base_exists + + - name: Check if docker-compose.staging.yml exists in staging stack + stat: + path: "{{ app_stack_path }}/docker-compose.staging.yml" + register: compose_override_exists + + - name: Fail if docker-compose files don't exist + fail: + msg: | + Staging Stack docker-compose files not found at {{ app_stack_path }} + + Required files: + - docker-compose.base.yml + - docker-compose.staging.yml + + The Staging Stack must be deployed first via: + ansible-playbook -i inventory/production.yml playbooks/setup-infrastructure.yml + + This will create the staging stack with docker-compose files and .env file. + when: + - not compose_base_exists.stat.exists or not compose_override_exists.stat.exists + + - name: Create backup directory + file: + path: "{{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + + tasks: + - name: Verify docker-compose files exist + stat: + path: "{{ app_stack_path }}/docker-compose.base.yml" + register: compose_base_check + + - name: Verify docker-compose.staging.yml exists + stat: + path: "{{ app_stack_path }}/docker-compose.staging.yml" + register: compose_override_check + + - name: Fail if docker-compose files don't exist + fail: + msg: | + Staging Stack docker-compose files not found at {{ app_stack_path }} + + Required files: + - docker-compose.base.yml + - docker-compose.staging.yml + + The Staging Stack must be deployed first via: + ansible-playbook -i inventory/production.yml playbooks/setup-infrastructure.yml + + This will create the staging stack with docker-compose files and .env file. + when: + - not compose_base_check.stat.exists or not compose_override_check.stat.exists + + - name: Backup current deployment metadata + shell: | + docker compose -f {{ app_stack_path }}/docker-compose.base.yml -f {{ app_stack_path }}/docker-compose.staging.yml ps --format json 2>/dev/null > {{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}/current_containers.json || true + docker compose -f {{ app_stack_path }}/docker-compose.base.yml -f {{ app_stack_path }}/docker-compose.staging.yml config 2>/dev/null > {{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}/docker-compose-config.yml || true + args: + executable: /bin/bash + changed_when: false + ignore_errors: yes + + - name: Login to Docker registry (if credentials provided) + community.docker.docker_login: + registry_url: "{{ docker_registry_url }}" + username: "{{ docker_registry_username }}" + password: "{{ docker_registry_password }}" + no_log: yes + ignore_errors: yes + when: + - docker_registry_username is defined + - docker_registry_password is defined + - docker_registry_username | length > 0 + - docker_registry_password | length > 0 + register: registry_login + + - name: Pull new Docker image + community.docker.docker_image: + name: "{{ app_image }}" + tag: "{{ image_tag }}" + source: pull + force_source: yes + register: image_pull + + - name: Verify image was pulled successfully + fail: + msg: "Failed to pull image {{ app_image }}:{{ image_tag }}" + when: image_pull.failed + + - name: Update docker-compose.staging.yml with new image tag (all services) + replace: + path: "{{ app_stack_path }}/docker-compose.staging.yml" + # Match both localhost:5000 and registry.michaelschiemer.de (or any registry URL) + regexp: '^(\s+image:\s+)(localhost:5000|registry\.michaelschiemer\.de|{{ docker_registry }})/{{ app_name }}:.*$' + replace: '\1{{ app_image }}:{{ image_tag }}' + register: compose_updated + + - name: Redeploy staging stack with new image + import_role: + name: application + vars: + application_sync_files: false + application_compose_recreate: "always" + application_remove_orphans: true + application_stack_path: "{{ app_stack_path }}" + application_compose_files: + - "{{ app_stack_path }}/docker-compose.base.yml" + - "{{ app_stack_path }}/docker-compose.staging.yml" + + - name: Get deployed image information + shell: | + docker compose -f {{ app_stack_path }}/docker-compose.base.yml -f {{ app_stack_path }}/docker-compose.staging.yml config | grep -E "^\s+image:" | head -1 | awk '{print $2}' || echo "unknown" + args: + executable: /bin/bash + register: deployed_image + changed_when: false + + - name: Record deployment metadata + copy: + content: | + Deployment Timestamp: {{ deployment_timestamp }} + Git Commit: {{ git_commit_sha }} + Image Tag: {{ image_tag }} + Deployed Image: {{ deployed_image.stdout }} + Image Pull: {{ 'SUCCESS' if image_pull.changed else 'SKIPPED (already exists)' }} + Stack Deploy: {{ 'UPDATED' if application_stack_changed else 'NO_CHANGE' }} + Health Status: {{ application_health_output if application_health_output != '' else 'All services healthy' }} + Health Check HTTP Status: {{ application_healthcheck_status }} + dest: "{{ backups_path }}/{{ deployment_timestamp | regex_replace(':', '-') }}/deployment_metadata.txt" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' + + - name: Cleanup old backups (keep last {{ max_rollback_versions | default(5) }}) + shell: | + cd {{ backups_path }} + ls -dt */ 2>/dev/null | tail -n +{{ max_rollback_versions | default(5) + 1 }} | xargs -r rm -rf + args: + executable: /bin/bash + changed_when: false + ignore_errors: yes + + post_tasks: + - name: Display deployment summary + debug: + msg: + - "=== Staging Deployment Summary ===" + - "Image: {{ app_image }}:{{ image_tag }}" + - "Commit: {{ git_commit_sha }}" + - "Timestamp: {{ deployment_timestamp }}" + - "Image Pull: {{ 'SUCCESS' if image_pull.changed else 'SKIPPED' }}" + - "Stack Deploy: {{ 'UPDATED' if application_stack_changed else 'NO_CHANGE' }}" + - "Health Output: {{ application_health_output if application_health_output != '' else 'All services healthy' }}" + - "Health Check HTTP Status: {{ application_healthcheck_status }}" + - "Health Check URL: {{ health_check_url | default('https://staging.michaelschiemer.de/health') }}" + - "" + - "Next: Verify application is healthy" diff --git a/deployment/ansible/playbooks/fix-staging-compose-secrets.yml b/deployment/ansible/playbooks/fix-staging-compose-secrets.yml new file mode 100644 index 00000000..fdc68789 --- /dev/null +++ b/deployment/ansible/playbooks/fix-staging-compose-secrets.yml @@ -0,0 +1,138 @@ +--- +- name: Fix Staging docker-compose.staging.yml with Redis Secrets + hosts: production + gather_facts: yes + become: no + + tasks: + - name: Check current docker-compose.staging.yml on server + shell: | + cd ~/deployment/stacks/staging + echo "=== Current staging-app environment (REDIS-related) ===" + grep -A 50 "staging-app:" docker-compose.staging.yml | grep -A 30 "environment:" | grep -E "(REDIS_|CACHE_|SESSION_)" || echo "No Redis env vars found" + echo "" + echo "=== Current secrets section for staging-app ===" + grep -A 10 "staging-app:" docker-compose.staging.yml | grep -A 15 "secrets:" || echo "Secrets section not found" + echo "" + echo "=== Secrets definitions at bottom ===" + tail -30 docker-compose.staging.yml | grep -A 10 "secrets:" || echo "Secrets definitions not found" + register: current_config + ignore_errors: yes + + - name: Display current configuration + debug: + msg: "{{ current_config.stdout_lines }}" + + - name: Get repository root path + shell: | + cd "{{ playbook_dir }}/../../.." + pwd + register: repo_root + changed_when: false + delegate_to: localhost + become: no + + - name: Display repository root + debug: + msg: "Repository root: {{ repo_root.stdout }}" + + - name: Check if docker-compose.staging.yml exists in repository + stat: + path: "{{ repo_root.stdout }}/docker-compose.staging.yml" + register: compose_file_stat + delegate_to: localhost + become: no + + - name: Read docker-compose.staging.yml from repository + slurp: + src: "{{ repo_root.stdout }}/docker-compose.staging.yml" + register: compose_file_content + when: compose_file_stat.stat.exists + delegate_to: localhost + become: no + + - name: Write docker-compose.staging.yml to server + copy: + content: "{{ compose_file_content.content | b64decode }}" + dest: "~/deployment/stacks/staging/docker-compose.staging.yml" + mode: '0644' + when: compose_file_stat.stat.exists + + - name: Fail if docker-compose.staging.yml not found + fail: + msg: "Could not find docker-compose.staging.yml at {{ repo_root.stdout }}/docker-compose.staging.yml. Please ensure the file exists in the repository root." + when: not compose_file_stat.stat.exists + + - name: Verify updated docker-compose.staging.yml on server + shell: | + cd ~/deployment/stacks/staging + echo "=== Updated staging-app environment (REDIS-related) ===" + grep -A 50 "staging-app:" docker-compose.staging.yml | grep -A 30 "environment:" | grep -E "(REDIS_|CACHE_|SESSION_|_FILE)" || echo "No Redis env vars found" + echo "" + echo "=== Updated secrets section for staging-app ===" + grep -A 10 "staging-app:" docker-compose.staging.yml | grep -A 15 "secrets:" || echo "Secrets section not found" + echo "" + echo "=== Secrets definitions at bottom ===" + tail -30 docker-compose.staging.yml | grep -A 10 "redis_password:" || echo "Secrets definitions not found" + register: updated_config + ignore_errors: yes + + - name: Display updated configuration + debug: + msg: "{{ updated_config.stdout_lines }}" + + - name: Restart staging containers to apply changes + shell: | + cd ~/deployment/stacks/staging + docker compose -f docker-compose.base.yml -f docker-compose.staging.yml up -d --force-recreate + register: restart_result + ignore_errors: yes + + - name: Display restart result + debug: + msg: "{{ restart_result.stdout_lines }}" + + - name: Wait for containers to start + pause: + seconds: 10 + + - name: Check container status after fix + shell: | + cd ~/deployment/stacks/staging + docker compose -f docker-compose.base.yml -f docker-compose.staging.yml ps + register: container_status + ignore_errors: yes + + - name: Display container status + debug: + msg: "{{ container_status.stdout_lines }}" + + - name: Verify REDIS_PASSWORD_FILE in container + shell: | + echo "=== Checking REDIS_PASSWORD_FILE in staging-app container ===" + docker exec staging-app env | grep REDIS_PASSWORD || echo "REDIS_PASSWORD variables not found" + docker exec staging-app env | grep "_FILE" | grep REDIS || echo "REDIS_PASSWORD_FILE not found" + echo "" + echo "=== Checking /run/secrets/redis_password ===" + docker exec staging-app ls -la /run/secrets/redis_password 2>&1 || echo "Secret file not found" + register: container_check + ignore_errors: yes + + - name: Display container verification + debug: + msg: "{{ container_check.stdout_lines }}" + + - name: Summary + debug: + msg: + - "========================================" + - "FIX SUMMARY" + - "========================================" + - "1. Updated docker-compose.staging.yml on server" + - "2. Restarted staging containers" + - "3. Verified REDIS_PASSWORD_FILE configuration" + - "" + - "Next steps:" + - "- Check staging-app logs: docker logs staging-app" + - "- Test Redis connection from staging-app container" + - "- Verify no more NOAUTH errors in logs" diff --git a/deployment/ansible/playbooks/fix-staging-secrets-permissions.yml b/deployment/ansible/playbooks/fix-staging-secrets-permissions.yml new file mode 100644 index 00000000..6723963e --- /dev/null +++ b/deployment/ansible/playbooks/fix-staging-secrets-permissions.yml @@ -0,0 +1,82 @@ +--- +- name: Fix Staging Secrets Permissions + hosts: production + gather_facts: yes + become: no + + tasks: + - name: Check secrets file permissions in staging-app container + shell: | + echo "=== Checking /run/secrets/redis_password permissions ===" + docker exec staging-app ls -la /run/secrets/redis_password 2>&1 || echo "File not found" + echo "" + echo "=== Checking /run/secrets directory permissions ===" + docker exec staging-app ls -la /run/secrets/ | head -10 + echo "" + echo "=== Current user ===" + docker exec staging-app whoami + echo "" + echo "=== Testing file read access ===" + docker exec staging-app cat /run/secrets/redis_password 2>&1 | head -c 20 || echo "Cannot read file" + echo "..." + register: permissions_check + ignore_errors: yes + + - name: Display permissions check + debug: + msg: "{{ permissions_check.stdout_lines }}" + + - name: Try to fix permissions via entrypoint modification + shell: | + cd ~/deployment/stacks/staging + # Check if staging-app has an entrypoint that can be modified + grep -A 5 "staging-app:" docker-compose.staging.yml | grep -A 10 "entrypoint:" | head -5 + register: entrypoint_check + ignore_errors: yes + + - name: Display entrypoint check + debug: + msg: "{{ entrypoint_check.stdout_lines }}" + + - name: Check if we can read secrets as root in container + shell: | + echo "=== Reading secret as root ===" + docker exec -u root staging-app cat /run/secrets/redis_password 2>&1 | head -c 20 || echo "Cannot read even as root" + echo "..." + echo "" + echo "=== Checking file owner ===" + docker exec -u root staging-app stat -c "%U:%G %a" /run/secrets/redis_password 2>&1 || echo "Cannot stat" + register: root_check + ignore_errors: yes + + - name: Display root check + debug: "{{ root_check.stdout_lines }}" + debug: + msg: "{{ root_check.stdout_lines }}" + + - name: Check container user configuration + shell: | + cd ~/deployment/stacks/staging + echo "=== staging-app user configuration ===" + grep -A 20 "staging-app:" docker-compose.staging.yml | grep -E "(user:|USER)" || echo "No user specified (defaults to www-data)" + register: user_config + ignore_errors: yes + + - name: Display user configuration + debug: + msg: "{{ user_config.stdout_lines }}" + + - name: Summary and Recommendations + debug: + msg: + - "========================================" + - "PERMISSIONS ISSUE ANALYSIS" + - "========================================" + - "The secret file exists but is not readable by the PHP process." + - "" + - "Possible solutions:" + - "1. Run PHP-FPM as root (NOT RECOMMENDED for security)" + - "2. Create a wrapper script that reads secrets as root and exports them" + - "3. Modify entrypoint to chmod/chown secrets (may not work on /run/secrets)" + - "4. Use environment variables instead of file-based secrets" + - "5. Modify docker-compose to use a different secrets mount path with proper permissions" diff --git a/deployment/stacks/semaphore/playbooks/deploy-staging.yml b/deployment/stacks/semaphore/playbooks/deploy-staging.yml index 831b2b56..62423f46 100644 --- a/deployment/stacks/semaphore/playbooks/deploy-staging.yml +++ b/deployment/stacks/semaphore/playbooks/deploy-staging.yml @@ -52,16 +52,15 @@ path: "{{ deployment_path }}/docker-compose.staging.yml" register: staging_compose_exists - - name: Copy docker-compose files if missing + - name: Copy docker-compose files (always update) copy: src: "{{ item.src }}" dest: "{{ deployment_path }}/{{ item.dest }}" mode: '0644' + force: yes loop: - { src: "docker-compose.base.yml", dest: "docker-compose.base.yml" } - { src: "docker-compose.staging.yml", dest: "docker-compose.staging.yml" } - when: not (item.dest == "docker-compose.base.yml" and base_compose_exists.stat.exists) or - not (item.dest == "docker-compose.staging.yml" and staging_compose_exists.stat.exists) delegate_to: localhost become: no diff --git a/deployment/stacks/staging/README.md b/deployment/stacks/staging/README.md index bb04c153..6504129c 100644 --- a/deployment/stacks/staging/README.md +++ b/deployment/stacks/staging/README.md @@ -55,37 +55,43 @@ DB_USERNAME=postgres DB_PASSWORD= # Redis (separate instance) -REDIS_PASSWORD= +# Note: REDIS_PASSWORD is loaded from Docker Secret via REDIS_PASSWORD_FILE +# See secrets/redis_password.txt file CACHE_PREFIX=staging # Git GIT_REPOSITORY_URL=https://git.michaelschiemer.de/michael/michaelschiemer.git GIT_BRANCH=staging -GIT_TOKEN= +# Note: GIT_TOKEN is loaded from Docker Secret via GIT_TOKEN_FILE +# See secrets/git_token.txt file ``` ## Deployment ### Initial Setup +The staging environment uses `docker-compose.staging.yml` in the repository root, which is used as an override for `docker-compose.base.yml`. + ```bash # Create staging stack directory on server mkdir -p ~/deployment/stacks/staging -# Copy docker-compose.yml from repository -cp deployment/stacks/staging/docker-compose.yml ~/deployment/stacks/staging/ +# Copy docker-compose files from repository +cp docker-compose.base.yml ~/deployment/stacks/staging/ +cp docker-compose.staging.yml ~/deployment/stacks/staging/ -# Create .env file -cd ~/deployment/stacks/staging -cp .env.example .env -# Edit .env with staging-specific values +# Create secrets directory and files +mkdir -p ~/deployment/stacks/staging/secrets +# Create secret files (redis_password.txt, db_user_password.txt, app_key.txt, etc.) +# These files should contain the actual secret values # Ensure networks exist docker network create traefik-public 2>/dev/null || true docker network create staging-internal 2>/dev/null || true # Start staging stack -docker compose up -d +cd ~/deployment/stacks/staging +docker compose -f docker-compose.base.yml -f docker-compose.staging.yml up -d ``` ### Auto-Deployment @@ -99,7 +105,7 @@ docker compose up -d ```bash # Check container status cd ~/deployment/stacks/staging -docker compose ps +docker compose -f docker-compose.base.yml -f docker-compose.staging.yml ps # Test staging URL curl https://staging.michaelschiemer.de/health @@ -150,7 +156,8 @@ docker logs staging-nginx ```bash # Force code pull in staging-app docker exec staging-app bash -c "cd /var/www/html && git pull origin staging" -docker compose restart staging-app staging-nginx +cd ~/deployment/stacks/staging +docker compose -f docker-compose.base.yml -f docker-compose.staging.yml restart staging-app staging-nginx ``` ## Cleanup @@ -159,6 +166,6 @@ To remove staging environment: ```bash cd ~/deployment/stacks/staging -docker compose down -v # Removes volumes too +docker compose -f docker-compose.base.yml -f docker-compose.staging.yml down -v # Removes volumes too docker network rm staging-internal ``` diff --git a/deployment/stacks/staging/docker-compose.yml b/deployment/stacks/staging/docker-compose.yml deleted file mode 100644 index 29e7bfbc..00000000 --- a/deployment/stacks/staging/docker-compose.yml +++ /dev/null @@ -1,356 +0,0 @@ -# Staging Environment - Docker Compose Configuration -# Separate stack for staging.michaelschiemer.de - -services: - # PHP-FPM Application Runtime - staging-app: - image: git.michaelschiemer.de:5000/framework:latest - container_name: staging-app - restart: unless-stopped - networks: - - staging-internal - environment: - - TZ=Europe/Berlin - - APP_ENV=staging - - APP_DEBUG=${APP_DEBUG:-true} - - APP_URL=https://staging.michaelschiemer.de - - APP_KEY=${APP_KEY:-} - # Git Repository - clones staging branch - - GIT_REPOSITORY_URL=${GIT_REPOSITORY_URL:-} - - GIT_BRANCH=staging - - GIT_TOKEN=${GIT_TOKEN:-} - - GIT_USERNAME=${GIT_USERNAME:-} - - GIT_PASSWORD=${GIT_PASSWORD:-} - # Database (can share with production or use separate) - - DB_HOST=${DB_HOST:-postgres} - - DB_PORT=${DB_PORT:-5432} - - DB_DATABASE=${DB_DATABASE:-michaelschiemer_staging} - - DB_USERNAME=${DB_USERNAME} - - DB_PASSWORD=${DB_PASSWORD} - # Redis - - REDIS_HOST=staging-redis - - REDIS_PORT=6379 - - REDIS_PASSWORD=${REDIS_PASSWORD} - # Cache - - CACHE_DRIVER=redis - - CACHE_PREFIX=${CACHE_PREFIX:-staging} - # Session - - SESSION_DRIVER=redis - - SESSION_LIFETIME=${SESSION_LIFETIME:-120} - # Queue - - QUEUE_DRIVER=redis - - QUEUE_CONNECTION=default - volumes: - - staging-code:/var/www/html - - staging-storage:/var/www/html/storage - - staging-logs:/var/www/html/storage/logs - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - # Override entrypoint to only start PHP-FPM (not nginx) + fix git ownership - entrypoint: ["/bin/sh", "-c"] - command: - - | - # Load secrets from /run/secrets/ - echo "πŸ” Loading secrets from /run/secrets/..." - [ -f /run/secrets/DB_PASSWORD ] && export DB_PASSWORD="$(cat /run/secrets/DB_PASSWORD)" || true - [ -f /run/secrets/APP_KEY ] && export APP_KEY="$(cat /run/secrets/APP_KEY)" || true - [ -f /run/secrets/GIT_TOKEN ] && export GIT_TOKEN="$(cat /run/secrets/GIT_TOKEN)" || true - - # Fix Git ownership issue - # Ensure Git treats the mounted repository as safe regardless of owner - git config --global --add safe.directory /var/www/html 2>/dev/null || true - git config --system --add safe.directory /var/www/html 2>/dev/null || true - - # Git Clone/Pull functionality - if [ -n "$GIT_REPOSITORY_URL" ]; then - echo "" - echo "πŸ“₯ Cloning/Pulling code from Git repository..." - - GIT_BRANCH="${GIT_BRANCH:-main}" - GIT_TARGET_DIR="/var/www/html" - - # Setup Git credentials - if [ -n "$GIT_TOKEN" ]; then - GIT_URL_WITH_AUTH=$(echo "$GIT_REPOSITORY_URL" | sed "s|https://|https://${GIT_TOKEN}@|") - elif [ -n "$GIT_USERNAME" ] && [ -n "$GIT_PASSWORD" ]; then - GIT_URL_WITH_AUTH=$(echo "$GIT_REPOSITORY_URL" | sed "s|https://|https://${GIT_USERNAME}:${GIT_PASSWORD}@|") - else - GIT_URL_WITH_AUTH="$GIT_REPOSITORY_URL" - fi - - # Clone or pull - if [ ! -d "$GIT_TARGET_DIR/.git" ]; then - echo "πŸ“₯ Cloning repository from $GIT_REPOSITORY_URL (branch: $GIT_BRANCH)..." - if [ "$(ls -A $GIT_TARGET_DIR 2>/dev/null)" ]; then - find "$GIT_TARGET_DIR" -mindepth 1 -maxdepth 1 ! -name "storage" -exec rm -rf {} \; 2>/dev/null || true - fi - TEMP_CLONE="${GIT_TARGET_DIR}.tmp" - rm -rf "$TEMP_CLONE" 2>/dev/null || true - if git -c safe.directory=/var/www/html clone --branch "$GIT_BRANCH" --depth 1 "$GIT_URL_WITH_AUTH" "$TEMP_CLONE"; then - find "$GIT_TARGET_DIR" -mindepth 1 -maxdepth 1 ! -name "storage" -exec rm -rf {} \; 2>/dev/null || true - find "$TEMP_CLONE" -mindepth 1 -maxdepth 1 ! -name "." ! -name ".." -exec mv {} "$GIT_TARGET_DIR/" \; 2>/dev/null || true - rm -rf "$TEMP_CLONE" 2>/dev/null || true - echo "βœ… Repository cloned successfully" - fi - else - echo "πŸ”„ Pulling latest changes from $GIT_BRANCH..." - cd "$GIT_TARGET_DIR" - git -c safe.directory=/var/www/html fetch origin "$GIT_BRANCH" || echo "⚠️ Git fetch failed" - git -c safe.directory=/var/www/html reset --hard "origin/$GIT_BRANCH" || echo "⚠️ Git reset failed" - git -c safe.directory=/var/www/html clean -fd || true - fi - - # Install dependencies - if [ -f "$GIT_TARGET_DIR/composer.json" ]; then - echo "πŸ“¦ Installing/updating Composer dependencies..." - cd "$GIT_TARGET_DIR" - composer install --no-dev --optimize-autoloader --no-interaction --no-scripts || echo "⚠️ Composer install failed" - composer dump-autoload --optimize --classmap-authoritative || true - fi - - echo "βœ… Git sync completed" - else - echo "" - echo "ℹ️ GIT_REPOSITORY_URL not set, using code from image" - fi - - echo "" - echo "πŸ“Š Environment variables:" - env | grep -E "DB_|APP_" | grep -v "PASSWORD|KEY|SECRET" || true - - echo "" - echo "πŸ› οΈ Adjusting filesystem permissions..." - chown -R www-data:www-data /var/www/html/storage /var/www/html/bootstrap/cache 2>/dev/null || true - find /var/www/html/storage /var/www/html/bootstrap/cache -type d -exec chmod 775 {} \; 2>/dev/null || true - find /var/www/html/storage /var/www/html/bootstrap/cache -type f -exec chmod 664 {} \; 2>/dev/null || true - - # Start PHP-FPM only (no nginx) - echo "" - echo "πŸš€ Starting PHP-FPM..." - exec php-fpm - healthcheck: - test: ["CMD-SHELL", "php-fpm-healthcheck || true"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s - depends_on: - staging-redis: - condition: service_started - - # Nginx Web Server - staging-nginx: - image: git.michaelschiemer.de:5000/framework:latest - container_name: staging-nginx - restart: unless-stopped - networks: - - traefik-public - - staging-internal - environment: - - TZ=Europe/Berlin - - APP_ENV=staging - - APP_DEBUG=${APP_DEBUG:-true} - # Git Repository - clones staging branch - - GIT_REPOSITORY_URL=${GIT_REPOSITORY_URL:-} - - GIT_BRANCH=staging - - GIT_TOKEN=${GIT_TOKEN:-} - - GIT_USERNAME=${GIT_USERNAME:-} - - GIT_PASSWORD=${GIT_PASSWORD:-} - volumes: - - ./nginx/conf.d:/etc/nginx/conf.d:ro - - staging-code:/var/www/html:ro - - staging-storage:/var/www/html/storage:ro - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - # Wait for code to be available (cloned by staging-app container) then start nginx - entrypoint: ["/bin/sh", "-c"] - command: - - | - # Wait for code to be available in shared volume (staging-app container clones it) - GIT_TARGET_DIR="/var/www/html" - echo "⏳ [staging-nginx] Waiting for code to be available in shared volume..." - for i in 1 2 3 4 5 6 7 8 9 10; do - if [ -d "$$GIT_TARGET_DIR/public" ]; then - echo "βœ… [staging-nginx] Code found in shared volume" - break - fi - echo " [staging-nginx] Waiting... ($$i/10)" - sleep 2 - done - - # If code still not available after wait, try to copy from image as fallback - if [ ! -d "$$GIT_TARGET_DIR/public" ] && [ -d "/var/www/html.orig" ]; then - echo "⚠️ [staging-nginx] Code not found in shared volume, copying from image..." - find /var/www/html.orig -mindepth 1 -maxdepth 1 ! -name "storage" -exec cp -r {} "$$GIT_TARGET_DIR/" \; 2>/dev/null || true - fi - - # Fix nginx upstream configuration - sites-enabled/default overrides conf.d/default.conf - # This is critical: nginx sites-available/default uses 127.0.0.1:9000 but PHP-FPM runs in staging-app container - if [ -f "/etc/nginx/sites-available/default" ]; then - echo "πŸ”§ [staging-nginx] Fixing PHP-FPM upstream configuration..." - # Replace in upstream block - sed -i '/upstream php-upstream {/,/}/s|server 127.0.0.1:9000;|server staging-app:9000;|g' /etc/nginx/sites-available/default || true - sed -i '/upstream php-upstream {/,/}/s|server localhost:9000;|server staging-app:9000;|g' /etc/nginx/sites-available/default || true - # Replace any direct fastcgi_pass references too - sed -i 's|fastcgi_pass 127.0.0.1:9000;|fastcgi_pass php-upstream;|g' /etc/nginx/sites-available/default || true - sed -i 's|fastcgi_pass localhost:9000;|fastcgi_pass php-upstream;|g' /etc/nginx/sites-available/default || true - echo "βœ… [staging-nginx] PHP-FPM upstream fixed" - fi - - # Start nginx only (no PHP-FPM, no Git clone - staging-app container handles that) - echo "πŸš€ [staging-nginx] Starting nginx..." - exec nginx -g "daemon off;" - labels: - - "traefik.enable=true" - # HTTP Router for staging subdomain - - "traefik.http.routers.staging.rule=Host(`staging.michaelschiemer.de`)" - - "traefik.http.routers.staging.entrypoints=websecure" - - "traefik.http.routers.staging.tls=true" - - "traefik.http.routers.staging.tls.certresolver=letsencrypt" - # Service - - "traefik.http.services.staging.loadbalancer.server.port=80" - # Middleware - - "traefik.http.routers.staging.middlewares=default-chain@file" - # Network - - "traefik.docker.network=traefik-public" - healthcheck: - test: ["CMD-SHELL", "curl -f http://127.0.0.1/health || exit 1"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - depends_on: - staging-app: - condition: service_started - - # Redis Cache/Session/Queue Backend (separate from production) - staging-redis: - image: redis:7-alpine - container_name: staging-redis - restart: unless-stopped - networks: - - staging-internal - environment: - - TZ=Europe/Berlin - command: > - redis-server - --requirepass ${REDIS_PASSWORD} - --maxmemory 256mb - --maxmemory-policy allkeys-lru - --save 900 1 - --save 300 10 - --save 60 10000 - --appendonly yes - --appendfsync everysec - volumes: - - staging-redis-data:/data - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - healthcheck: - test: ["CMD", "redis-cli", "--raw", "incr", "ping"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 10s - - # Queue Worker (Background Jobs) - staging-queue-worker: - image: git.michaelschiemer.de:5000/framework:latest - container_name: staging-queue-worker - restart: unless-stopped - networks: - - staging-internal - environment: - - TZ=Europe/Berlin - - APP_ENV=staging - - APP_DEBUG=${APP_DEBUG:-true} - # Database - - DB_HOST=${DB_HOST:-postgres} - - DB_PORT=${DB_PORT:-5432} - - DB_DATABASE=${DB_DATABASE:-michaelschiemer_staging} - - DB_USERNAME=${DB_USERNAME} - - DB_PASSWORD=${DB_PASSWORD} - # Redis - - REDIS_HOST=staging-redis - - REDIS_PORT=6379 - - REDIS_PASSWORD=${REDIS_PASSWORD} - # Queue - - QUEUE_DRIVER=redis - - QUEUE_CONNECTION=default - - QUEUE_WORKER_SLEEP=${QUEUE_WORKER_SLEEP:-3} - - QUEUE_WORKER_TRIES=${QUEUE_WORKER_TRIES:-3} - - QUEUE_WORKER_TIMEOUT=${QUEUE_WORKER_TIMEOUT:-60} - volumes: - - staging-code:/var/www/html - - staging-storage:/var/www/html/storage - - staging-logs:/var/www/html/storage/logs - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - command: php console.php queue:work --queue=default --timeout=${QUEUE_WORKER_TIMEOUT:-60} - healthcheck: - test: ["CMD-SHELL", "php -r 'exit(0);' && test -f /var/www/html/console.php || exit 1"] - interval: 60s - timeout: 10s - retries: 3 - start_period: 30s - depends_on: - staging-app: - condition: service_started - staging-redis: - condition: service_started - - # Scheduler (Cron Jobs) - staging-scheduler: - image: git.michaelschiemer.de:5000/framework:latest - container_name: staging-scheduler - restart: unless-stopped - networks: - - staging-internal - environment: - - TZ=Europe/Berlin - - APP_ENV=staging - - APP_DEBUG=${APP_DEBUG:-true} - # Database - - DB_HOST=${DB_HOST:-postgres} - - DB_PORT=${DB_PORT:-5432} - - DB_DATABASE=${DB_DATABASE:-michaelschiemer_staging} - - DB_USERNAME=${DB_USERNAME} - - DB_PASSWORD=${DB_PASSWORD} - # Redis - - REDIS_HOST=staging-redis - - REDIS_PORT=6379 - - REDIS_PASSWORD=${REDIS_PASSWORD} - volumes: - - staging-code:/var/www/html - - staging-storage:/var/www/html/storage - - staging-logs:/var/www/html/storage/logs - - /etc/timezone:/etc/timezone:ro - - /etc/localtime:/etc/localtime:ro - command: php console.php scheduler:run - healthcheck: - test: ["CMD-SHELL", "php -r 'exit(0);' && test -f /var/www/html/console.php || exit 1"] - interval: 60s - timeout: 10s - retries: 3 - start_period: 30s - depends_on: - staging-app: - condition: service_started - staging-redis: - condition: service_started - -volumes: - staging-code: - name: staging-code - staging-storage: - name: staging-storage - staging-logs: - name: staging-logs - staging-redis-data: - name: staging-redis-data - -networks: - traefik-public: - external: true - staging-internal: - driver: bridge diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index b6342548..e8de0532 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -48,11 +48,13 @@ services: - QUEUE_DRIVER=redis - QUEUE_CONNECTION=default # Use Docker Secrets via *_FILE pattern (Framework supports this automatically) - - DB_PASSWORD_FILE=/run/secrets/db_user_password - - REDIS_PASSWORD_FILE=/run/secrets/redis_password - - APP_KEY_FILE=/run/secrets/app_key - - VAULT_ENCRYPTION_KEY_FILE=/run/secrets/vault_encryption_key - - GIT_TOKEN_FILE=/run/secrets/git_token + # Note: These paths will be set by the entrypoint script after copying secrets + # to /var/www/html/storage/secrets/ for www-data access + - DB_PASSWORD_FILE=/var/www/html/storage/secrets/db_user_password + - REDIS_PASSWORD_FILE=/var/www/html/storage/secrets/redis_password + - APP_KEY_FILE=/var/www/html/storage/secrets/app_key + - VAULT_ENCRYPTION_KEY_FILE=/var/www/html/storage/secrets/vault_encryption_key + - GIT_TOKEN_FILE=/var/www/html/storage/secrets/git_token volumes: - staging-code:/var/www/html - staging-storage:/var/www/html/storage @@ -70,6 +72,55 @@ services: command: - | + # Copy Docker Secrets to readable location for www-data + # Docker Secrets are only readable by root, but PHP (www-data) needs to read them. + # We copy them here as root to a location where www-data can read them. + echo "πŸ” Setting up Docker Secrets for PHP access..." + SECRETS_DIR="/var/www/html/storage/secrets" + mkdir -p "$SECRETS_DIR" + chmod 750 "$SECRETS_DIR" + chown www-data:www-data "$SECRETS_DIR" + + if [ -f /run/secrets/redis_password ]; then + cp /run/secrets/redis_password "$SECRETS_DIR/redis_password" 2>/dev/null || true + chmod 640 "$SECRETS_DIR/redis_password" + chown www-data:www-data "$SECRETS_DIR/redis_password" + export REDIS_PASSWORD_FILE="$SECRETS_DIR/redis_password" + echo "βœ… Copied redis_password to $SECRETS_DIR/redis_password" + fi + + if [ -f /run/secrets/db_user_password ]; then + cp /run/secrets/db_user_password "$SECRETS_DIR/db_user_password" 2>/dev/null || true + chmod 640 "$SECRETS_DIR/db_user_password" + chown www-data:www-data "$SECRETS_DIR/db_user_password" + export DB_PASSWORD_FILE="$SECRETS_DIR/db_user_password" + echo "βœ… Copied db_user_password to $SECRETS_DIR/db_user_password" + fi + + if [ -f /run/secrets/app_key ]; then + cp /run/secrets/app_key "$SECRETS_DIR/app_key" 2>/dev/null || true + chmod 640 "$SECRETS_DIR/app_key" + chown www-data:www-data "$SECRETS_DIR/app_key" + export APP_KEY_FILE="$SECRETS_DIR/app_key" + echo "βœ… Copied app_key to $SECRETS_DIR/app_key" + fi + + if [ -f /run/secrets/vault_encryption_key ]; then + cp /run/secrets/vault_encryption_key "$SECRETS_DIR/vault_encryption_key" 2>/dev/null || true + chmod 640 "$SECRETS_DIR/vault_encryption_key" + chown www-data:www-data "$SECRETS_DIR/vault_encryption_key" + export VAULT_ENCRYPTION_KEY_FILE="$SECRETS_DIR/vault_encryption_key" + echo "βœ… Copied vault_encryption_key to $SECRETS_DIR/vault_encryption_key" + fi + + if [ -f /run/secrets/git_token ]; then + cp /run/secrets/git_token "$SECRETS_DIR/git_token" 2>/dev/null || true + chmod 640 "$SECRETS_DIR/git_token" + chown www-data:www-data "$SECRETS_DIR/git_token" + export GIT_TOKEN_FILE="$SECRETS_DIR/git_token" + echo "βœ… Copied git_token to $SECRETS_DIR/git_token" + fi + # Fix Git ownership issue # Ensure Git treats the mounted repository as safe regardless of owner git config --global --add safe.directory /var/www/html 2>/dev/null || true @@ -138,9 +189,14 @@ services: find /var/www/html/storage /var/www/html/bootstrap/cache -type d -exec chmod 775 {} \; 2>/dev/null || true find /var/www/html/storage /var/www/html/bootstrap/cache -type f -exec chmod 664 {} \; 2>/dev/null || true + # Keep PHP-FPM secure with clear_env = yes (default) + # The *_FILE environment variables are passed explicitly via docker-compose environment section + # PHP's DockerSecretsResolver will read the secrets from the files specified in *_FILE vars + # Start PHP-FPM only (no nginx) echo "" echo "πŸš€ Starting PHP-FPM..." + echo "REDIS_PASSWORD_FILE: ${REDIS_PASSWORD_FILE:-NOT SET}" exec php-fpm healthcheck: test: ["CMD-SHELL", "php-fpm-healthcheck || true"] diff --git a/src/Framework/Redis/RedisConnection.php b/src/Framework/Redis/RedisConnection.php index 6f6e63a0..8aa6591a 100644 --- a/src/Framework/Redis/RedisConnection.php +++ b/src/Framework/Redis/RedisConnection.php @@ -112,7 +112,7 @@ final class RedisConnection implements RedisConnectionInterface $this->connected = false; throw new RedisConnectionException( - "Failed to connect to Redis ({$this->name}): " . $e->getMessage() . " with Host: {$this->config->host} and Password: {$this->config->password}", + "Failed to connect to Redis ({$this->name}): " . $e->getMessage() . " with Host: {$this->config->host} and Password: {$this->config->password} should be {$_ENV['REDIS_HOST']} and {$_ENV['REDIS_PASSWORD']}", previous: $e ); }