michaelschiemer/deployment/applications/scripts/health-check.sh

#!/bin/bash

# Health Check Script for Custom PHP Framework
# Verifies application health after deployment
# Usage: ./health-check.sh [environment] [options]

set -euo pipefail

# Script configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../../" && pwd)"
DEPLOYMENT_DIR="${PROJECT_ROOT}/deployment"
APPLICATIONS_DIR="${DEPLOYMENT_DIR}/applications"

# Default settings
DEFAULT_ENV="staging"
VERBOSE=false
MAX_RETRIES=10
RETRY_INTERVAL=30

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging functions
log() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] INFO: $1${NC}"
}

warn() {
    echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARN: $1${NC}"
}

error() {
    echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR: $1${NC}"
}

debug() {
    if [ "$VERBOSE" = true ]; then
        echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')] DEBUG: $1${NC}"
    fi
}

success() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] SUCCESS: $1${NC}"
}

# Usage information
show_usage() {
    cat << EOF
Usage: $0 [environment] [options]

Environment:
    staging      Check staging environment (default)
    production   Check production environment

Options:
    --verbose           Enable verbose output
    --max-retries N     Maximum number of health check retries (default: 10)
    --retry-interval N  Seconds between retries (default: 30)
    -h, --help          Show this help message

Examples:
    $0 staging                              # Check staging environment
    $0 production --verbose                 # Check production with detailed output
    $0 staging --max-retries 5              # Check with custom retry limit

EOF
}

# Parse command line arguments
parse_arguments() {
    local environment=""
    
    while [[ $# -gt 0 ]]; do
        case $1 in
            staging|production)
                environment="$1"
                shift
                ;;
            --verbose)
                VERBOSE=true
                shift
                ;;
            --max-retries)
                MAX_RETRIES="$2"
                shift 2
                ;;
            --retry-interval)
                RETRY_INTERVAL="$2"
                shift 2
                ;;
            -h|--help)
                show_usage
                exit 0
                ;;
            *)
                error "Unknown argument: $1"
                show_usage
                exit 1
                ;;
        esac
    done
    
    # Set environment, defaulting to staging
    CHECK_ENV="${environment:-$DEFAULT_ENV}"
}

# Load environment configuration
load_environment() {
    local env_file="${APPLICATIONS_DIR}/environments/.env.${CHECK_ENV}"
    
    if [[ ! -f "$env_file" ]]; then
        error "Environment file not found: $env_file"
        exit 1
    fi
    
    debug "Loading environment from: $env_file"
    set -a
    source "$env_file"
    set +a
}

# Check Docker container health
check_container_health() {
    log "Checking Docker container health"
    
    local compose_files="-f ${PROJECT_ROOT}/docker-compose.yml -f ${APPLICATIONS_DIR}/docker-compose.${CHECK_ENV}.yml"
    local env_file="--env-file ${APPLICATIONS_DIR}/environments/.env.${CHECK_ENV}"
    
    cd "$PROJECT_ROOT"
    
    # Get container status
    local containers
    containers=$(docker-compose $compose_files $env_file ps --services)
    
    local all_healthy=true
    
    while IFS= read -r service; do
        if [[ -n "$service" ]]; then
            local container_status
            container_status=$(docker-compose $compose_files $env_file ps "$service" | tail -n +3 | awk '{print $4}')
            
            case "$container_status" in
                "Up"|"Up (healthy)")
                    success "Container $service: $container_status"
                    ;;
                *)
                    error "Container $service: $container_status"
                    all_healthy=false
                    ;;
            esac
        fi
    done <<< "$containers"
    
    if [ "$all_healthy" = false ]; then
        return 1
    fi
    
    debug "All containers are healthy"
    return 0
}

# Check HTTP endpoints
check_http_endpoints() {
    log "Checking HTTP endpoints"
    
    # Determine base URL based on environment
    local base_url
    if [ "$CHECK_ENV" = "production" ]; then
        base_url="https://michaelschiemer.de"
    else
        base_url="https://localhost:${APP_SSL_PORT:-443}"
    fi
    
    # Test endpoints
    local endpoints=(
        "/"
        "/health"
        "/api/health"
    )
    
    local all_ok=true
    
    for endpoint in "${endpoints[@]}"; do
        local url="${base_url}${endpoint}"
        debug "Testing endpoint: $url"
        
        local http_code
        http_code=$(curl -s -o /dev/null -w "%{http_code}" -k \
                        -H "User-Agent: Mozilla/5.0 (HealthCheck)" \
                        --max-time 30 \
                        "$url" || echo "000")
        
        case "$http_code" in
            200|301|302)
                success "Endpoint $endpoint: HTTP $http_code"
                ;;
            000)
                error "Endpoint $endpoint: Connection failed"
                all_ok=false
                ;;
            *)
                error "Endpoint $endpoint: HTTP $http_code"
                all_ok=false
                ;;
        esac
    done
    
    if [ "$all_ok" = false ]; then
        return 1
    fi
    
    debug "All HTTP endpoints are responding correctly"
    return 0
}

# Check database connectivity
check_database() {
    log "Checking database connectivity"
    
    local compose_files="-f ${PROJECT_ROOT}/docker-compose.yml -f ${APPLICATIONS_DIR}/docker-compose.${CHECK_ENV}.yml"
    local env_file="--env-file ${APPLICATIONS_DIR}/environments/.env.${CHECK_ENV}"
    
    cd "$PROJECT_ROOT"
    
    # Test database connection through application
    local db_check
    if db_check=$(docker-compose $compose_files $env_file exec -T php \
                    php console.php db:ping 2>&1); then
        success "Database connection: OK"
        debug "Database response: $db_check"
        return 0
    else
        error "Database connection: FAILED"
        error "Database error: $db_check"
        return 1
    fi
}

# Check Redis connectivity
check_redis() {
    log "Checking Redis connectivity"
    
    local compose_files="-f ${PROJECT_ROOT}/docker-compose.yml -f ${APPLICATIONS_DIR}/docker-compose.${CHECK_ENV}.yml"
    local env_file="--env-file ${APPLICATIONS_DIR}/environments/.env.${CHECK_ENV}"
    
    cd "$PROJECT_ROOT"
    
    # Test Redis connection
    local redis_check
    if redis_check=$(docker-compose $compose_files $env_file exec -T redis \
                       redis-cli ping 2>&1); then
        if [[ "$redis_check" == *"PONG"* ]]; then
            success "Redis connection: OK"
            debug "Redis response: $redis_check"
            return 0
        else
            error "Redis connection: Unexpected response - $redis_check"
            return 1
        fi
    else
        error "Redis connection: FAILED"
        error "Redis error: $redis_check"
        return 1
    fi
}

# Check queue worker status
check_queue_worker() {
    log "Checking queue worker status"
    
    local compose_files="-f ${PROJECT_ROOT}/docker-compose.yml -f ${APPLICATIONS_DIR}/docker-compose.${CHECK_ENV}.yml"
    local env_file="--env-file ${APPLICATIONS_DIR}/environments/.env.${CHECK_ENV}"
    
    cd "$PROJECT_ROOT"
    
    # Check if worker container is running
    local worker_status
    worker_status=$(docker-compose $compose_files $env_file ps queue-worker | tail -n +3 | awk '{print $4}')
    
    case "$worker_status" in
        "Up")
            success "Queue worker: Running"
            
            # Check worker process inside container
            local worker_process
            if worker_process=$(docker-compose $compose_files $env_file exec -T queue-worker \
                                  ps aux | grep -v grep | grep worker 2>&1); then
                debug "Worker process found: $worker_process"
                return 0
            else
                warn "Queue worker container running but no worker process found"
                return 1
            fi
            ;;
        *)
            error "Queue worker: $worker_status"
            return 1
            ;;
    esac
}

# Check application performance
check_performance() {
    log "Checking application performance"
    
    local base_url
    if [ "$CHECK_ENV" = "production" ]; then
        base_url="https://michaelschiemer.de"
    else
        base_url="https://localhost:${APP_SSL_PORT:-443}"
    fi
    
    # Test response times
    local response_time
    response_time=$(curl -s -o /dev/null -w "%{time_total}" -k \
                      -H "User-Agent: Mozilla/5.0 (HealthCheck)" \
                      --max-time 30 \
                      "${base_url}/" || echo "timeout")
    
    if [[ "$response_time" == "timeout" ]]; then
        error "Performance check: Request timed out"
        return 1
    fi
    
    local response_ms
    response_ms=$(echo "$response_time * 1000" | bc -l | cut -d. -f1)
    
    # Performance thresholds (milliseconds)
    local warning_threshold=2000
    local error_threshold=5000
    
    if [ "$response_ms" -lt "$warning_threshold" ]; then
        success "Performance: ${response_ms}ms (Good)"
        return 0
    elif [ "$response_ms" -lt "$error_threshold" ]; then
        warn "Performance: ${response_ms}ms (Slow)"
        return 0
    else
        error "Performance: ${response_ms}ms (Too slow)"
        return 1
    fi
}

# Check SSL certificate
check_ssl() {
    if [ "$CHECK_ENV" != "production" ]; then
        debug "Skipping SSL check for non-production environment"
        return 0
    fi
    
    log "Checking SSL certificate"
    
    local domain="michaelschiemer.de"
    local ssl_info
    
    if ssl_info=$(echo | openssl s_client -connect "${domain}:443" -servername "$domain" 2>/dev/null | \
                    openssl x509 -noout -dates 2>/dev/null); then
        
        local expiry_date
        expiry_date=$(echo "$ssl_info" | grep "notAfter" | cut -d= -f2)
        
        local expiry_timestamp
        expiry_timestamp=$(date -d "$expiry_date" +%s 2>/dev/null || echo "0")
        
        local current_timestamp
        current_timestamp=$(date +%s)
        
        local days_until_expiry
        days_until_expiry=$(( (expiry_timestamp - current_timestamp) / 86400 ))
        
        if [ "$days_until_expiry" -gt 30 ]; then
            success "SSL certificate: Valid (expires in $days_until_expiry days)"
            return 0
        elif [ "$days_until_expiry" -gt 7 ]; then
            warn "SSL certificate: Expires soon (in $days_until_expiry days)"
            return 0
        else
            error "SSL certificate: Critical expiry (in $days_until_expiry days)"
            return 1
        fi
    else
        error "SSL certificate: Could not retrieve certificate information"
        return 1
    fi
}

# Comprehensive health check with retries
run_health_checks() {
    log "Starting comprehensive health check for $CHECK_ENV environment"
    
    local checks=(
        "check_container_health"
        "check_database"
        "check_redis"
        "check_queue_worker"
        "check_http_endpoints"
        "check_performance"
        "check_ssl"
    )
    
    local attempt=1
    local all_passed=false
    
    while [ $attempt -le $MAX_RETRIES ] && [ "$all_passed" = false ]; do
        log "Health check attempt $attempt of $MAX_RETRIES"
        
        local failed_checks=()
        
        for check in "${checks[@]}"; do
            debug "Running check: $check"
            if ! $check; then
                failed_checks+=("$check")
            fi
        done
        
        if [ ${#failed_checks[@]} -eq 0 ]; then
            all_passed=true
            success "All health checks passed!"
        else
            warn "Failed checks: ${failed_checks[*]}"
            
            if [ $attempt -lt $MAX_RETRIES ]; then
                log "Waiting $RETRY_INTERVAL seconds before retry..."
                sleep $RETRY_INTERVAL
            fi
        fi
        
        ((attempt++))
    done
    
    if [ "$all_passed" = false ]; then
        error "Health checks failed after $MAX_RETRIES attempts"
        return 1
    fi
    
    return 0
}

# Main function
main() {
    log "Starting health check for $CHECK_ENV environment"
    
    load_environment
    
    if run_health_checks; then
        success "Health check completed successfully for $CHECK_ENV environment"
        log "Application is healthy and ready to serve traffic"
        return 0
    else
        error "Health check failed for $CHECK_ENV environment"
        error "Please review the application status and fix any issues"
        return 1
    fi
}

# Parse arguments and run main function
parse_arguments "$@"
main