michael/michaelschiemer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: staging deployment configuration and redis secrets handling

Browse Source
This commit is contained in:
Michael Schiemer
2025-11-03 00:15:43 +01:00
parent ff572534e9
commit 7a2cb0b63e
12 changed files with 1027 additions and 388 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
deployment/stacks/staging/README.md
View File

@@ -55,37 +55,43 @@ DB_USERNAME=postgres
DB_PASSWORD=<password>
# Redis (separate instance)
REDIS_PASSWORD=<password>
# Note: REDIS_PASSWORD is loaded from Docker Secret via REDIS_PASSWORD_FILE
# See secrets/redis_password.txt file
CACHE_PREFIX=staging
# Git
GIT_REPOSITORY_URL=https://git.michaelschiemer.de/michael/michaelschiemer.git
GIT_BRANCH=staging
GIT_TOKEN=<token>
# Note: GIT_TOKEN is loaded from Docker Secret via GIT_TOKEN_FILE
# See secrets/git_token.txt file
```
## Deployment
### Initial Setup
The staging environment uses `docker-compose.staging.yml` in the repository root, which is used as an override for `docker-compose.base.yml`.
```bash
# Create staging stack directory on server
mkdir -p ~/deployment/stacks/staging
# Copy docker-compose.yml from repository
cp deployment/stacks/staging/docker-compose.yml ~/deployment/stacks/staging/
# Copy docker-compose files from repository
cp docker-compose.base.yml ~/deployment/stacks/staging/
cp docker-compose.staging.yml ~/deployment/stacks/staging/
# Create .env file
cd ~/deployment/stacks/staging
cp .env.example .env
# Edit .env with staging-specific values
# Create secrets directory and files
mkdir -p ~/deployment/stacks/staging/secrets
# Create secret files (redis_password.txt, db_user_password.txt, app_key.txt, etc.)
# These files should contain the actual secret values
# Ensure networks exist
docker network create traefik-public 2>/dev/null || true
docker network create staging-internal 2>/dev/null || true
# Start staging stack
docker compose up -d
cd ~/deployment/stacks/staging
docker compose -f docker-compose.base.yml -f docker-compose.staging.yml up -d
```
### Auto-Deployment
@@ -99,7 +105,7 @@ docker compose up -d
```bash
# Check container status
cd ~/deployment/stacks/staging
docker compose ps
docker compose -f docker-compose.base.yml -f docker-compose.staging.yml ps
# Test staging URL
curl https://staging.michaelschiemer.de/health
@@ -150,7 +156,8 @@ docker logs staging-nginx
```bash
# Force code pull in staging-app
docker exec staging-app bash -c "cd /var/www/html && git pull origin staging"
docker compose restart staging-app staging-nginx
cd ~/deployment/stacks/staging
docker compose -f docker-compose.base.yml -f docker-compose.staging.yml restart staging-app staging-nginx
```
## Cleanup
@@ -159,6 +166,6 @@ To remove staging environment:
```bash
cd ~/deployment/stacks/staging
docker compose down -v # Removes volumes too
docker compose -f docker-compose.base.yml -f docker-compose.staging.yml down -v # Removes volumes too
docker network rm staging-internal
```

356
deployment/stacks/staging/docker-compose.yml
View File

@@ -1,356 +0,0 @@
# Staging Environment - Docker Compose Configuration
# Separate stack for staging.michaelschiemer.de
services:
# PHP-FPM Application Runtime
staging-app:
image: git.michaelschiemer.de:5000/framework:latest
container_name: staging-app
restart: unless-stopped
networks:
- staging-internal
environment:
- TZ=Europe/Berlin
- APP_ENV=staging
- APP_DEBUG=${APP_DEBUG:-true}
- APP_URL=https://staging.michaelschiemer.de
- APP_KEY=${APP_KEY:-}
# Git Repository - clones staging branch
- GIT_REPOSITORY_URL=${GIT_REPOSITORY_URL:-}
- GIT_BRANCH=staging
- GIT_TOKEN=${GIT_TOKEN:-}
- GIT_USERNAME=${GIT_USERNAME:-}
- GIT_PASSWORD=${GIT_PASSWORD:-}
# Database (can share with production or use separate)
- DB_HOST=${DB_HOST:-postgres}
- DB_PORT=${DB_PORT:-5432}
- DB_DATABASE=${DB_DATABASE:-michaelschiemer_staging}
- DB_USERNAME=${DB_USERNAME}
- DB_PASSWORD=${DB_PASSWORD}
# Redis
- REDIS_HOST=staging-redis
- REDIS_PORT=6379
- REDIS_PASSWORD=${REDIS_PASSWORD}
# Cache
- CACHE_DRIVER=redis
- CACHE_PREFIX=${CACHE_PREFIX:-staging}
# Session
- SESSION_DRIVER=redis
- SESSION_LIFETIME=${SESSION_LIFETIME:-120}
# Queue
- QUEUE_DRIVER=redis
- QUEUE_CONNECTION=default
volumes:
- staging-code:/var/www/html
- staging-storage:/var/www/html/storage
- staging-logs:/var/www/html/storage/logs
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
# Override entrypoint to only start PHP-FPM (not nginx) + fix git ownership
entrypoint: ["/bin/sh", "-c"]
command:
- |
# Load secrets from /run/secrets/
echo "🔐 Loading secrets from /run/secrets/..."
[ -f /run/secrets/DB_PASSWORD ] && export DB_PASSWORD="$(cat /run/secrets/DB_PASSWORD)" || true
[ -f /run/secrets/APP_KEY ] && export APP_KEY="$(cat /run/secrets/APP_KEY)" || true
[ -f /run/secrets/GIT_TOKEN ] && export GIT_TOKEN="$(cat /run/secrets/GIT_TOKEN)" || true
# Fix Git ownership issue
# Ensure Git treats the mounted repository as safe regardless of owner
git config --global --add safe.directory /var/www/html 2>/dev/null || true
git config --system --add safe.directory /var/www/html 2>/dev/null || true
# Git Clone/Pull functionality
if [ -n "$GIT_REPOSITORY_URL" ]; then
echo ""
echo "📥 Cloning/Pulling code from Git repository..."
GIT_BRANCH="${GIT_BRANCH:-main}"
GIT_TARGET_DIR="/var/www/html"
# Setup Git credentials
if [ -n "$GIT_TOKEN" ]; then
GIT_URL_WITH_AUTH=$(echo "$GIT_REPOSITORY_URL" | sed "s|https://|https://${GIT_TOKEN}@|")
elif [ -n "$GIT_USERNAME" ] && [ -n "$GIT_PASSWORD" ]; then
GIT_URL_WITH_AUTH=$(echo "$GIT_REPOSITORY_URL" | sed "s|https://|https://${GIT_USERNAME}:${GIT_PASSWORD}@|")
else
GIT_URL_WITH_AUTH="$GIT_REPOSITORY_URL"
fi
# Clone or pull
if [ ! -d "$GIT_TARGET_DIR/.git" ]; then
echo "📥 Cloning repository from $GIT_REPOSITORY_URL (branch: $GIT_BRANCH)..."
if [ "$(ls -A $GIT_TARGET_DIR 2>/dev/null)" ]; then
find "$GIT_TARGET_DIR" -mindepth 1 -maxdepth 1 ! -name "storage" -exec rm -rf {} \; 2>/dev/null || true
fi
TEMP_CLONE="${GIT_TARGET_DIR}.tmp"
rm -rf "$TEMP_CLONE" 2>/dev/null || true
if git -c safe.directory=/var/www/html clone --branch "$GIT_BRANCH" --depth 1 "$GIT_URL_WITH_AUTH" "$TEMP_CLONE"; then
find "$GIT_TARGET_DIR" -mindepth 1 -maxdepth 1 ! -name "storage" -exec rm -rf {} \; 2>/dev/null || true
find "$TEMP_CLONE" -mindepth 1 -maxdepth 1 ! -name "." ! -name ".." -exec mv {} "$GIT_TARGET_DIR/" \; 2>/dev/null || true
rm -rf "$TEMP_CLONE" 2>/dev/null || true
echo "✅ Repository cloned successfully"
fi
else
echo "🔄 Pulling latest changes from $GIT_BRANCH..."
cd "$GIT_TARGET_DIR"
git -c safe.directory=/var/www/html fetch origin "$GIT_BRANCH" || echo "⚠️ Git fetch failed"
git -c safe.directory=/var/www/html reset --hard "origin/$GIT_BRANCH" || echo "⚠️ Git reset failed"
git -c safe.directory=/var/www/html clean -fd || true
fi
# Install dependencies
if [ -f "$GIT_TARGET_DIR/composer.json" ]; then
echo "📦 Installing/updating Composer dependencies..."
cd "$GIT_TARGET_DIR"
composer install --no-dev --optimize-autoloader --no-interaction --no-scripts || echo "⚠️ Composer install failed"
composer dump-autoload --optimize --classmap-authoritative || true
fi
echo "✅ Git sync completed"
else
echo ""
echo "ℹ️ GIT_REPOSITORY_URL not set, using code from image"
fi
echo ""
echo "📊 Environment variables:"
env | grep -E "DB_|APP_" | grep -v "PASSWORD|KEY|SECRET" || true
echo ""
echo "🛠️ Adjusting filesystem permissions..."
chown -R www-data:www-data /var/www/html/storage /var/www/html/bootstrap/cache 2>/dev/null || true
find /var/www/html/storage /var/www/html/bootstrap/cache -type d -exec chmod 775 {} \; 2>/dev/null || true
find /var/www/html/storage /var/www/html/bootstrap/cache -type f -exec chmod 664 {} \; 2>/dev/null || true
# Start PHP-FPM only (no nginx)
echo ""
echo "🚀 Starting PHP-FPM..."
exec php-fpm
healthcheck:
test: ["CMD-SHELL", "php-fpm-healthcheck || true"]
interval: 30s
timeout: 10s
retries: 3
start_period: 40s
depends_on:
staging-redis:
condition: service_started
# Nginx Web Server
staging-nginx:
image: git.michaelschiemer.de:5000/framework:latest
container_name: staging-nginx
restart: unless-stopped
networks:
- traefik-public
- staging-internal
environment:
- TZ=Europe/Berlin
- APP_ENV=staging
- APP_DEBUG=${APP_DEBUG:-true}
# Git Repository - clones staging branch
- GIT_REPOSITORY_URL=${GIT_REPOSITORY_URL:-}
- GIT_BRANCH=staging
- GIT_TOKEN=${GIT_TOKEN:-}
- GIT_USERNAME=${GIT_USERNAME:-}
- GIT_PASSWORD=${GIT_PASSWORD:-}
volumes:
- ./nginx/conf.d:/etc/nginx/conf.d:ro
- staging-code:/var/www/html:ro
- staging-storage:/var/www/html/storage:ro
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
# Wait for code to be available (cloned by staging-app container) then start nginx
entrypoint: ["/bin/sh", "-c"]
command:
- |
# Wait for code to be available in shared volume (staging-app container clones it)
GIT_TARGET_DIR="/var/www/html"
echo "⏳ [staging-nginx] Waiting for code to be available in shared volume..."
for i in 1 2 3 4 5 6 7 8 9 10; do
if [ -d "$$GIT_TARGET_DIR/public" ]; then
echo "✅ [staging-nginx] Code found in shared volume"
break
fi
echo " [staging-nginx] Waiting... ($$i/10)"
sleep 2
done
# If code still not available after wait, try to copy from image as fallback
if [ ! -d "$$GIT_TARGET_DIR/public" ] && [ -d "/var/www/html.orig" ]; then
echo "⚠️ [staging-nginx] Code not found in shared volume, copying from image..."
find /var/www/html.orig -mindepth 1 -maxdepth 1 ! -name "storage" -exec cp -r {} "$$GIT_TARGET_DIR/" \; 2>/dev/null || true
fi
# Fix nginx upstream configuration - sites-enabled/default overrides conf.d/default.conf
# This is critical: nginx sites-available/default uses 127.0.0.1:9000 but PHP-FPM runs in staging-app container
if [ -f "/etc/nginx/sites-available/default" ]; then
echo "🔧 [staging-nginx] Fixing PHP-FPM upstream configuration..."
# Replace in upstream block
sed -i '/upstream php-upstream {/,/}/s|server 127.0.0.1:9000;|server staging-app:9000;|g' /etc/nginx/sites-available/default || true
sed -i '/upstream php-upstream {/,/}/s|server localhost:9000;|server staging-app:9000;|g' /etc/nginx/sites-available/default || true
# Replace any direct fastcgi_pass references too
sed -i 's|fastcgi_pass 127.0.0.1:9000;|fastcgi_pass php-upstream;|g' /etc/nginx/sites-available/default || true
sed -i 's|fastcgi_pass localhost:9000;|fastcgi_pass php-upstream;|g' /etc/nginx/sites-available/default || true
echo "✅ [staging-nginx] PHP-FPM upstream fixed"
fi
# Start nginx only (no PHP-FPM, no Git clone - staging-app container handles that)
echo "🚀 [staging-nginx] Starting nginx..."
exec nginx -g "daemon off;"
labels:
- "traefik.enable=true"
# HTTP Router for staging subdomain
- "traefik.http.routers.staging.rule=Host(`staging.michaelschiemer.de`)"
- "traefik.http.routers.staging.entrypoints=websecure"
- "traefik.http.routers.staging.tls=true"
- "traefik.http.routers.staging.tls.certresolver=letsencrypt"
# Service
- "traefik.http.services.staging.loadbalancer.server.port=80"
# Middleware
- "traefik.http.routers.staging.middlewares=default-chain@file"
# Network
- "traefik.docker.network=traefik-public"
healthcheck:
test: ["CMD-SHELL", "curl -f http://127.0.0.1/health || exit 1"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
depends_on:
staging-app:
condition: service_started
# Redis Cache/Session/Queue Backend (separate from production)
staging-redis:
image: redis:7-alpine
container_name: staging-redis
restart: unless-stopped
networks:
- staging-internal
environment:
- TZ=Europe/Berlin
command: >
redis-server
--requirepass ${REDIS_PASSWORD}
--maxmemory 256mb
--maxmemory-policy allkeys-lru
--save 900 1
--save 300 10
--save 60 10000
--appendonly yes
--appendfsync everysec
volumes:
- staging-redis-data:/data
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
healthcheck:
test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
interval: 30s
timeout: 10s
retries: 3
start_period: 10s
# Queue Worker (Background Jobs)
staging-queue-worker:
image: git.michaelschiemer.de:5000/framework:latest
container_name: staging-queue-worker
restart: unless-stopped
networks:
- staging-internal
environment:
- TZ=Europe/Berlin
- APP_ENV=staging
- APP_DEBUG=${APP_DEBUG:-true}
# Database
- DB_HOST=${DB_HOST:-postgres}
- DB_PORT=${DB_PORT:-5432}
- DB_DATABASE=${DB_DATABASE:-michaelschiemer_staging}
- DB_USERNAME=${DB_USERNAME}
- DB_PASSWORD=${DB_PASSWORD}
# Redis
- REDIS_HOST=staging-redis
- REDIS_PORT=6379
- REDIS_PASSWORD=${REDIS_PASSWORD}
# Queue
- QUEUE_DRIVER=redis
- QUEUE_CONNECTION=default
- QUEUE_WORKER_SLEEP=${QUEUE_WORKER_SLEEP:-3}
- QUEUE_WORKER_TRIES=${QUEUE_WORKER_TRIES:-3}
- QUEUE_WORKER_TIMEOUT=${QUEUE_WORKER_TIMEOUT:-60}
volumes:
- staging-code:/var/www/html
- staging-storage:/var/www/html/storage
- staging-logs:/var/www/html/storage/logs
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
command: php console.php queue:work --queue=default --timeout=${QUEUE_WORKER_TIMEOUT:-60}
healthcheck:
test: ["CMD-SHELL", "php -r 'exit(0);' && test -f /var/www/html/console.php || exit 1"]
interval: 60s
timeout: 10s
retries: 3
start_period: 30s
depends_on:
staging-app:
condition: service_started
staging-redis:
condition: service_started
# Scheduler (Cron Jobs)
staging-scheduler:
image: git.michaelschiemer.de:5000/framework:latest
container_name: staging-scheduler
restart: unless-stopped
networks:
- staging-internal
environment:
- TZ=Europe/Berlin
- APP_ENV=staging
- APP_DEBUG=${APP_DEBUG:-true}
# Database
- DB_HOST=${DB_HOST:-postgres}
- DB_PORT=${DB_PORT:-5432}
- DB_DATABASE=${DB_DATABASE:-michaelschiemer_staging}
- DB_USERNAME=${DB_USERNAME}
- DB_PASSWORD=${DB_PASSWORD}
# Redis
- REDIS_HOST=staging-redis
- REDIS_PORT=6379
- REDIS_PASSWORD=${REDIS_PASSWORD}
volumes:
- staging-code:/var/www/html
- staging-storage:/var/www/html/storage
- staging-logs:/var/www/html/storage/logs
- /etc/timezone:/etc/timezone:ro
- /etc/localtime:/etc/localtime:ro
command: php console.php scheduler:run
healthcheck:
test: ["CMD-SHELL", "php -r 'exit(0);' && test -f /var/www/html/console.php || exit 1"]
interval: 60s
timeout: 10s
retries: 3
start_period: 30s
depends_on:
staging-app:
condition: service_started
staging-redis:
condition: service_started
volumes:
staging-code:
name: staging-code
staging-storage:
name: staging-storage
staging-logs:
name: staging-logs
staging-redis-data:
name: staging-redis-data
networks:
traefik-public:
external: true
staging-internal:
driver: bridge