diff --git a/README.md b/README.md index c2bb07f0..717b23fa 100644 --- a/README.md +++ b/README.md @@ -320,3 +320,4 @@ make setup # Deployment (Code + Compose auf Server bringen) make deploy +# CI/CD Pipeline Test - Fri Nov 7 08:54:41 PM CET 2025 diff --git a/deployment/ansible/playbooks/README.md b/deployment/ansible/playbooks/README.md index f0c1f8ca..06203d06 100644 --- a/deployment/ansible/playbooks/README.md +++ b/deployment/ansible/playbooks/README.md @@ -47,6 +47,7 @@ - **`setup-gitea-runner-ci.yml`** - Gitea Runner CI Setup - **`setup-gitea-initial-config.yml`** - Gitea Initial Setup (automatisiert via app.ini + CLI) - **`setup-gitea-repository.yml`** - Erstellt Repository in Gitea und konfiguriert Git-Remote (automatisiert via API) +- **`update-gitea-config.yml`** - Aktualisiert Gitea-Konfiguration (Cache, Connection Pooling) zur Behebung von Performance-Problemen - **`install-docker.yml`** - Docker Installation auf Server ## Entfernte/Legacy Playbooks diff --git a/deployment/ansible/playbooks/update-gitea-config.yml b/deployment/ansible/playbooks/update-gitea-config.yml index 71fa7900..fda77b3a 100644 --- a/deployment/ansible/playbooks/update-gitea-config.yml +++ b/deployment/ansible/playbooks/update-gitea-config.yml @@ -1,49 +1,134 @@ --- -- name: Update Gitea Configuration and Restart - hosts: production - become: no - gather_facts: yes +# Ansible Playbook: Update Gitea Configuration +# Purpose: Update Gitea app.ini configuration to fix performance issues +# Usage: +# ansible-playbook -i inventory/production.yml playbooks/update-gitea-config.yml \ +# --vault-password-file secrets/.vault_pass +- name: Update Gitea Configuration + hosts: production vars: gitea_stack_path: "{{ stacks_base_path }}/gitea" + gitea_url: "https://{{ gitea_domain }}" + gitea_app_ini_path: "{{ gitea_stack_path }}/app.ini" + gitea_app_ini_container_path: "/data/gitea/conf/app.ini" tasks: - - name: Copy updated docker-compose.yml to production server - copy: - src: "{{ playbook_dir }}/../../stacks/gitea/docker-compose.yml" - dest: "{{ gitea_stack_path }}/docker-compose.yml" - owner: "{{ ansible_user }}" - group: "{{ ansible_user }}" - mode: '0644' - - - name: Restart Gitea stack with updated configuration - community.docker.docker_compose_v2: - project_src: "{{ gitea_stack_path }}" - state: present - pull: never - recreate: always - remove_orphans: no - register: gitea_restart - - - name: Wait for Gitea to be ready - wait_for: - timeout: 60 - when: gitea_restart.changed - - - name: Verify Gitea Actions configuration + - name: Verify Gitea container exists shell: | - docker exec gitea cat /data/gitea/conf/app.ini 2>/dev/null | grep -A 3 "\[actions\]" || echo "Config not accessible" - register: gitea_config + docker compose -f {{ gitea_stack_path }}/docker-compose.yml ps gitea | grep -q "gitea" + register: gitea_exists changed_when: false + failed_when: false + + - name: Fail if Gitea container does not exist + fail: + msg: "Gitea container does not exist. Please deploy Gitea stack first." + when: gitea_exists.rc != 0 + + - name: Get database configuration from environment + shell: | + docker compose -f {{ gitea_stack_path }}/docker-compose.yml exec -T gitea env | grep -E "^GITEA__database__" || true + register: gitea_db_env + changed_when: false + failed_when: false + + - name: Parse database configuration + set_fact: + gitea_db_type: "{{ (gitea_db_env.stdout | default('') | regex_search('GITEA__database__DB_TYPE=([^\n]+)', '\\1') or ['postgres']) | first }}" + gitea_db_host: "{{ (gitea_db_env.stdout | default('') | regex_search('GITEA__database__HOST=([^\n]+)', '\\1') or ['postgres:5432']) | first }}" + gitea_db_name: "{{ (gitea_db_env.stdout | default('') | regex_search('GITEA__database__NAME=([^\n]+)', '\\1') or ['gitea']) | first }}" + gitea_db_user: "{{ (gitea_db_env.stdout | default('') | regex_search('GITEA__database__USER=([^\n]+)', '\\1') or ['gitea']) | first }}" + gitea_db_passwd: "{{ (gitea_db_env.stdout | default('') | regex_search('GITEA__database__PASSWD=([^\n]+)', '\\1') or ['gitea_password']) | first }}" + + - name: Get Gitea server configuration from environment + shell: | + docker compose -f {{ gitea_stack_path }}/docker-compose.yml exec -T gitea env | grep -E "^GITEA__server__" || true + register: gitea_server_env + changed_when: false + failed_when: false + + - name: Parse server configuration + set_fact: + gitea_domain_parsed: "{{ (gitea_server_env.stdout | default('') | regex_search('GITEA__server__DOMAIN=([^\n]+)', '\\1') or [gitea_domain | default('git.michaelschiemer.de')]) | first }}" + ssh_port_parsed: "{{ (gitea_server_env.stdout | default('') | regex_search('GITEA__server__SSH_PORT=([^\n]+)', '\\1') or ['2222']) | first }}" + + - name: Set final configuration variables + set_fact: + gitea_domain: "{{ gitea_domain_parsed }}" + ssh_port: "{{ ssh_port_parsed }}" + ssh_listen_port: "{{ ssh_port_parsed }}" + + - name: Extract database host and port + set_fact: + gitea_db_hostname: "{{ gitea_db_host.split(':')[0] }}" + gitea_db_port: "{{ (gitea_db_host.split(':')[1]) | default('5432') }}" + + - name: Set Redis password + set_fact: + redis_password: "{{ vault_gitea_redis_password | default(vault_redis_password | default('gitea_redis_password')) }}" + + - name: Generate app.ini from template + template: + src: ../templates/gitea-app.ini.j2 + dest: "{{ gitea_app_ini_path }}" + mode: '0644' + vars: + postgres_db: "{{ gitea_db_name }}" + postgres_user: "{{ gitea_db_user }}" + postgres_password: "{{ gitea_db_passwd }}" + gitea_domain: "{{ gitea_domain }}" + ssh_port: "{{ ssh_port }}" + ssh_listen_port: "{{ ssh_listen_port }}" + disable_registration: true + redis_password: "{{ redis_password }}" + + - name: Copy app.ini to Gitea container + shell: | + docker compose -f {{ gitea_stack_path }}/docker-compose.yml cp {{ gitea_app_ini_path }} gitea:{{ gitea_app_ini_container_path }} ignore_errors: yes - - name: Display Gitea Actions configuration + - name: Wait for container to be ready for exec + shell: | + docker compose -f {{ gitea_stack_path }}/docker-compose.yml exec -T gitea true + register: container_ready + until: container_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false + + - name: Set correct permissions on app.ini in container + shell: | + docker compose -f {{ gitea_stack_path }}/docker-compose.yml exec -T --user git gitea chown 1000:1000 {{ gitea_app_ini_container_path }} && \ + docker compose -f {{ gitea_stack_path }}/docker-compose.yml exec -T --user git gitea chmod 644 {{ gitea_app_ini_container_path }} + + - name: Restart Gitea container + shell: | + docker compose -f {{ gitea_stack_path }}/docker-compose.yml restart gitea + + - name: Wait for Gitea to be ready after restart + uri: + url: "{{ gitea_url }}/api/healthz" + method: GET + status_code: [200] + validate_certs: false + timeout: 10 + register: gitea_health_after_restart + until: gitea_health_after_restart.status == 200 + retries: 30 + delay: 5 + changed_when: false + + - name: Display success message debug: - msg: - - "=== Gitea Configuration Update Complete ===" - - "Container restarted: {{ 'Yes' if gitea_restart.changed else 'No' }}" - - "" - - "Current Actions configuration:" - - "{{ gitea_config.stdout if gitea_config.stdout else 'Could not read config (container may still be starting)' }}" - - "" - - "The DEFAULT_ACTIONS_URL should now point to your Gitea instance instead of GitHub." + msg: | + Gitea configuration has been updated successfully! + + Changes applied: + - Redis cache enabled (persistent, survives container restarts) + - Redis sessions enabled (better performance and scalability) + - Redis queue enabled (persistent job processing) + - Database connection pooling configured + - Connection limits set to prevent "Connection reset by peer" errors + + Gitea should now be more stable and perform better with Redis. diff --git a/deployment/ansible/secrets/production.vault.yml.example b/deployment/ansible/secrets/production.vault.yml.example index ccf54281..5b6baa62 100644 --- a/deployment/ansible/secrets/production.vault.yml.example +++ b/deployment/ansible/secrets/production.vault.yml.example @@ -30,6 +30,16 @@ vault_git_token: "change-me-gitea-personal-access-token" # vault_git_username: "your-gitea-username" # vault_git_password: "your-gitea-password" +# Gitea Admin Credentials (for initial setup) +# Required for automated Gitea initial configuration +vault_gitea_admin_username: "admin" +vault_gitea_admin_password: "change-me-secure-gitea-admin-password" +vault_gitea_admin_email: "kontakt@michaelschiemer.de" + +# Gitea Redis Credentials +# Required for Redis cache, sessions, and queue +vault_gitea_redis_password: "change-me-secure-gitea-redis-password" + # Optional: Additional Secrets vault_encryption_key: "change-me-encryption-key" vault_session_secret: "change-me-session-secret" diff --git a/deployment/ansible/templates/gitea-app.ini.j2 b/deployment/ansible/templates/gitea-app.ini.j2 index b10e11ae..f9fb5fbe 100644 --- a/deployment/ansible/templates/gitea-app.ini.j2 +++ b/deployment/ansible/templates/gitea-app.ini.j2 @@ -21,13 +21,22 @@ HTTP_ADDR = 0.0.0.0 HTTP_PORT = 3000 ROOT_URL = https://{{ gitea_domain }}/ PUBLIC_URL_DETECTION = auto +;; Performance settings for handling concurrent requests +LFS_START_SERVER = true +LFS_CONTENT_PATH = data/lfs +LFS_JWT_SECRET = +;; Increase timeouts for better stability under load +READ_TIMEOUT = 60s +WRITE_TIMEOUT = 60s ;; SSH Configuration +;; Note: SSH_LISTEN_PORT should match the port exposed in docker-compose.yml +;; If SSH is not needed, set DISABLE_SSH = true and START_SSH_SERVER = false DISABLE_SSH = false -START_SSH_SERVER = true +START_SSH_SERVER = false SSH_DOMAIN = {{ gitea_domain }} -SSH_PORT = 22 -SSH_LISTEN_PORT = 22 +SSH_PORT = {{ ssh_port | default(2222) }} +SSH_LISTEN_PORT = {{ ssh_listen_port | default(2222) }} ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Database Configuration @@ -39,30 +48,45 @@ NAME = {{ postgres_db | default('gitea') }} USER = {{ postgres_user | default('gitea') }} PASSWD = {{ postgres_password | default('gitea_password') }} SSL_MODE = disable +;; Connection Pool Settings - Prevents "Connection reset by peer" errors +;; Increased limits for handling concurrent requests +MAX_OPEN_CONNS = 200 +MAX_IDLE_CONNS = 50 +CONN_MAX_LIFETIME = 600 +CONN_MAX_IDLE_TIME = 300 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Cache Configuration ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; [cache] -ENABLED = false -ADAPTER = memory +ENABLED = true +ADAPTER = redis +HOST = redis:6379 +PASSWORD = {{ redis_password | default('gitea_redis_password') }} +DB = 0 +;; Redis cache for better performance and persistence +;; Cache survives container restarts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Session Configuration ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; [session] -PROVIDER = file -PROVIDER_CONFIG = data/sessions +PROVIDER = redis +PROVIDER_CONFIG = network=tcp,addr=redis:6379,password={{ redis_password | default('gitea_redis_password') }},db=0,pool_size=100,idle_timeout=180 COOKIE_SECURE = true COOKIE_NAME = i_like_gitea GC_INTERVAL_TIME = 86400 SESSION_LIFE_TIME = 86400 +;; Redis sessions for better performance and scalability ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Queue Configuration ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; [queue] -TYPE = channel +TYPE = redis +CONN_STR = redis://:{{ redis_password | default('gitea_redis_password') }}@redis:6379/0 +;; Redis queue for persistent job processing +;; Jobs survive container restarts ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Service Configuration @@ -79,3 +103,10 @@ ENABLED = true ;; Do NOT set DEFAULT_ACTIONS_URL to a custom URL - it's not supported ;; Leaving it unset or setting to "self" will use the current instance ;DEFAULT_ACTIONS_URL = self + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Security Configuration +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +[security] +;; Set INSTALL_LOCK to true to skip the initial setup page +INSTALL_LOCK = true \ No newline at end of file diff --git a/deployment/stacks/gitea/README.md b/deployment/stacks/gitea/README.md index c5b3bc92..c24df6a1 100644 --- a/deployment/stacks/gitea/README.md +++ b/deployment/stacks/gitea/README.md @@ -411,17 +411,54 @@ docker compose logs redis ### Performance Issues -```bash -# Check MySQL slow queries -docker exec gitea-mysql tail -100 /var/log/mysql/slow-queries.log +**If Gitea has frequent outages or connection issues:** -# Analyze MySQL performance -docker exec gitea-mysql mysql -u root -p$MYSQL_ROOT_PASSWORD \ - -e "SHOW PROCESSLIST;" +1. **Update Gitea Configuration** (Recommended): + ```bash + cd deployment/ansible + ansible-playbook -i inventory/production.yml \ + playbooks/update-gitea-config.yml \ + --vault-password-file secrets/.vault_pass + ``` + + This playbook will: + - Enable Redis cache for better performance and persistence + - Configure database connection pooling + - Set connection limits to prevent "Connection reset by peer" errors -# Check Redis memory usage -docker exec gitea-redis redis-cli -a $REDIS_PASSWORD INFO memory -``` +2. **Manual Troubleshooting**: + ```bash + # Check PostgreSQL slow queries + docker exec gitea-postgres psql -U gitea -d gitea -c "SELECT * FROM pg_stat_activity;" + + # Check container resource usage + docker stats gitea gitea-postgres gitea-redis + + # Check Gitea logs for errors + docker compose logs --tail 100 gitea | grep -i error + + # Check Redis connection + docker exec gitea-redis redis-cli -a $REDIS_PASSWORD ping + ``` + +### Known Issues + +**Bad Gateway after many rapid requests (15-20 reloads):** +- **Status**: Known issue, non-critical +- **Symptoms**: Gitea returns "Bad Gateway" after 15-20 rapid page reloads, recovers after a few seconds +- **Impact**: Low - Gitea is functional for normal usage +- **Possible causes**: + - Container restart during high load + - Connection pool exhaustion (mitigated with increased limits) + - Traefik service discovery delay in host network mode +- **Workarounds**: + - Wait a few seconds and retry + - Use Redis cache (already enabled) for better performance + - Consider adding rate limiting if needed (see Traefik middlewares) +- **Future improvements**: + - Monitor and optimize connection pool usage + - Consider adding rate limiting middleware for Gitea + - Investigate Traefik service discovery in host network mode ### Reset Admin Password diff --git a/deployment/stacks/gitea/docker-compose.yml b/deployment/stacks/gitea/docker-compose.yml index 03f8b0e5..5d684def 100644 --- a/deployment/stacks/gitea/docker-compose.yml +++ b/deployment/stacks/gitea/docker-compose.yml @@ -5,6 +5,7 @@ services: restart: unless-stopped depends_on: - postgres + - redis networks: - traefik-public - gitea-internal @@ -18,10 +19,14 @@ services: - GITEA__database__NAME=${POSTGRES_DB:-gitea} - GITEA__database__USER=${POSTGRES_USER:-gitea} - GITEA__database__PASSWD=${POSTGRES_PASSWORD:-gitea_password} - - GITEA__cache__ENABLED=false - - GITEA__cache__ADAPTER=memory - - GITEA__session__PROVIDER=file - - GITEA__queue__TYPE=channel + - GITEA__cache__ENABLED=true + - GITEA__cache__ADAPTER=redis + - GITEA__cache__HOST=redis:6379 + - GITEA__cache__PASSWORD=${REDIS_PASSWORD:-gitea_redis_password} + - GITEA__session__PROVIDER=redis + - GITEA__session__PROVIDER_CONFIG=network=tcp,addr=redis:6379,password=${REDIS_PASSWORD:-gitea_redis_password},db=0,pool_size=100,idle_timeout=180 + - GITEA__queue__TYPE=redis + - GITEA__queue__CONN_STR=redis://:${REDIS_PASSWORD:-gitea_redis_password}@redis:6379/0 - GITEA__server__DOMAIN=${GITEA_DOMAIN:-git.michaelschiemer.de} - GITEA__server__ROOT_URL=https://${GITEA_DOMAIN:-git.michaelschiemer.de}/ - GITEA__server__SSH_DOMAIN=${GITEA_DOMAIN:-git.michaelschiemer.de} @@ -32,8 +37,6 @@ services: - gitea-data:/data - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro - ports: - - "2222:22" # SSH for Git operations labels: - "traefik.enable=true" @@ -47,6 +50,8 @@ services: # Service - "traefik.http.services.gitea.loadbalancer.server.port=3000" + # Use container name explicitly for host network mode + - "traefik.http.services.gitea.loadbalancer.server.scheme=http" # Middleware - "traefik.http.routers.gitea.middlewares=default-chain@file" @@ -68,6 +73,7 @@ services: - POSTGRES_DB=gitea - POSTGRES_USER=gitea - POSTGRES_PASSWORD=gitea_password + command: postgres -c max_connections=300 volumes: - postgres-data:/var/lib/postgresql/data healthcheck: @@ -77,32 +83,36 @@ services: retries: 3 start_period: 30s - # redis (disabled for now; Gitea configured to not use redis) - # redis: - # image: redis:7 - # container_name: gitea-redis - # restart: unless-stopped - # networks: - # - gitea-internal - # environment: - # - TZ=Europe/Berlin - # volumes: - # - redis-data:/data - # command: redis-server --appendonly yes - # healthcheck: - # test: ["CMD", "redis-cli", "ping"] - # interval: 30s - # timeout: 10s - # retries: 3 - # start_period: 10s + redis: + image: redis:7-alpine + container_name: gitea-redis + restart: unless-stopped + networks: + - gitea-internal + environment: + - TZ=Europe/Berlin + command: > + redis-server + --requirepass ${REDIS_PASSWORD:-gitea_redis_password} + --appendonly yes + --maxmemory 512mb + --maxmemory-policy allkeys-lru + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s volumes: gitea-data: name: gitea-data postgres-data: name: gitea-postgres-data - # redis-data: - # name: gitea-redis-data + redis-data: + name: gitea-redis-data networks: traefik-public: diff --git a/deployment/stacks/traefik/dynamic/gitea.yml b/deployment/stacks/traefik/dynamic/gitea.yml index 5ea073b1..c201aee7 100644 --- a/deployment/stacks/traefik/dynamic/gitea.yml +++ b/deployment/stacks/traefik/dynamic/gitea.yml @@ -1,15 +1,20 @@ -http: - routers: - gitea: - rule: Host(`git.michaelschiemer.de`) - entrypoints: - - websecure - service: gitea - tls: - certResolver: letsencrypt - priority: 100 - services: - gitea: - loadBalancer: - servers: - - url: http://gitea:3000 +# Gitea configuration is now handled via Docker labels in docker-compose.yml +# This file is kept for reference but is not used +# Traefik will automatically discover Gitea via Docker labels and use the container IP +# when running in host network mode +# +# http: +# routers: +# gitea: +# rule: Host(`git.michaelschiemer.de`) +# entrypoints: +# - websecure +# service: gitea +# tls: +# certResolver: letsencrypt +# priority: 100 +# services: +# gitea: +# loadBalancer: +# servers: +# - url: http://gitea:3000