Enable Discovery debug logging for production troubleshooting

- Add DISCOVERY_LOG_LEVEL=debug
- Add DISCOVERY_SHOW_PROGRESS=true
- Temporary changes for debugging InitializerProcessor fixes on production
This commit is contained in:
2025-08-11 20:13:26 +02:00
parent 59fd3dd3b1
commit 55a330b223
3683 changed files with 2956207 additions and 16948 deletions

View File

@@ -0,0 +1,40 @@
<?php
declare(strict_types=1);
namespace App\Framework\ErrorAggregation\Alerting;
use App\Framework\ErrorAggregation\AlertUrgency;
use App\Framework\ErrorAggregation\ErrorEvent;
use App\Framework\ErrorAggregation\ErrorPattern;
/**
* Interface for alert notification channels
*/
interface AlertChannel
{
/**
* Gets the channel name (email, slack, sms, etc.)
*/
public function getName(): string;
/**
* Sends an alert for an error pattern
*/
public function sendAlert(ErrorPattern $pattern, ErrorEvent $triggeringEvent, array $context = []): bool;
/**
* Checks if this channel can handle the given urgency level
*/
public function canHandle(AlertUrgency $urgency): bool;
/**
* Gets the delivery status of the last sent alert
*/
public function getLastDeliveryStatus(): array;
/**
* Tests the channel connectivity
*/
public function test(): bool;
}

View File

@@ -0,0 +1,371 @@
<?php
declare(strict_types=1);
namespace App\Framework\ErrorAggregation\Alerting;
use App\Framework\Cache\Cache;
use App\Framework\DateTime\Clock;
use App\Framework\ErrorAggregation\AlertUrgency;
use App\Framework\ErrorAggregation\ErrorEvent;
use App\Framework\ErrorAggregation\ErrorPattern;
use App\Framework\Logging\Logger;
use App\Framework\Queue\Queue;
/**
* Manages alert distribution across multiple channels
*/
final readonly class AlertManager
{
private const THROTTLE_CACHE_PREFIX = 'alert_throttle:';
private const ESCALATION_CACHE_PREFIX = 'alert_escalation:';
/**
* @var array<string, AlertChannel>
*/
private array $channels;
/**
* @var array<AlertUrgency, array<string>>
*/
private array $channelsByUrgency;
public function __construct(
private Cache $cache,
private Clock $clock,
private Queue $retryQueue,
private ?Logger $logger = null,
array $channels = [],
private array $throttleConfig = [],
private array $escalationConfig = []
) {
$this->channels = [];
$this->channelsByUrgency = [];
foreach ($channels as $channel) {
$this->addChannel($channel);
}
}
/**
* Adds an alert channel
*/
public function addChannel(AlertChannel $channel): void
{
$this->channels[$channel->getName()] = $channel;
// Map channels to urgency levels they can handle
foreach (AlertUrgency::cases() as $urgency) {
if ($channel->canHandle($urgency)) {
$this->channelsByUrgency[$urgency->value][] = $channel->getName();
}
}
}
/**
* Sends alert for error pattern
*/
public function sendAlert(ErrorPattern $pattern, ErrorEvent $triggeringEvent, array $context = []): array
{
$urgency = $pattern->getAlertUrgency();
$results = [];
// Check throttling
if ($this->isThrottled($pattern, $urgency)) {
$this->log('info', "Alert throttled for pattern {$pattern->id->toString()}", [
'pattern_id' => $pattern->id->toString(),
'urgency' => $urgency->value,
]);
return ['status' => 'throttled', 'channels' => []];
}
// Get channels for this urgency level
$channelNames = $this->channelsByUrgency[$urgency->value] ?? [];
if (empty($channelNames)) {
$this->log('warning', "No channels configured for urgency level {$urgency->value}");
return ['status' => 'no_channels', 'channels' => []];
}
$sentChannels = [];
$failedChannels = [];
foreach ($channelNames as $channelName) {
$channel = $this->channels[$channelName];
try {
$success = $channel->sendAlert($pattern, $triggeringEvent, $context);
if ($success) {
$sentChannels[] = $channelName;
$this->log('info', "Alert sent via {$channelName}", [
'pattern_id' => $pattern->id->toString(),
'channel' => $channelName,
]);
} else {
$failedChannels[] = $channelName;
$this->scheduleRetry($pattern, $triggeringEvent, $channelName, $context);
}
$results[$channelName] = [
'status' => $success ? 'sent' : 'failed',
'delivery_status' => $channel->getLastDeliveryStatus(),
];
} catch (\Throwable $e) {
$failedChannels[] = $channelName;
$this->scheduleRetry($pattern, $triggeringEvent, $channelName, $context);
$results[$channelName] = [
'status' => 'error',
'error' => $e->getMessage(),
];
$this->log('error', "Alert failed for channel {$channelName}: {$e->getMessage()}", [
'pattern_id' => $pattern->id->toString(),
'channel' => $channelName,
'exception' => $e,
]);
}
}
// Update throttling
$this->updateThrottle($pattern, $urgency);
// Schedule escalation if needed
if (! empty($failedChannels) && $urgency->requiresImmediateProcessing()) {
$this->scheduleEscalation($pattern, $triggeringEvent, $failedChannels, $context);
}
return [
'status' => empty($failedChannels) ? 'success' : 'partial',
'channels' => $results,
'sent' => $sentChannels,
'failed' => $failedChannels,
];
}
/**
* Processes alert retry from queue
*/
public function processRetry(array $retryData): bool
{
$patternData = $retryData['pattern'];
$eventData = $retryData['event'];
$channelName = $retryData['channel'];
$context = $retryData['context'] ?? [];
$attempt = $retryData['attempt'] ?? 1;
if (! isset($this->channels[$channelName])) {
$this->log('error', "Channel {$channelName} not found for retry");
return false;
}
$pattern = ErrorPattern::fromArray($patternData);
$event = ErrorEvent::fromArray($eventData);
$channel = $this->channels[$channelName];
try {
$success = $channel->sendAlert($pattern, $event, $context);
if ($success) {
$this->log('info', "Retry successful for channel {$channelName}", [
'pattern_id' => $pattern->id->toString(),
'attempt' => $attempt,
]);
return true;
} else {
$urgency = $pattern->getAlertUrgency();
$retryStrategy = $urgency->getRetryStrategy();
if ($attempt < $retryStrategy['attempts']) {
// Schedule next retry
$nextDelay = $retryStrategy['delays'][$attempt] ?? 300;
$this->scheduleRetry($pattern, $event, $channelName, $context, $attempt + 1, $nextDelay);
$this->log('info', "Retry {$attempt} failed, scheduling attempt " . ($attempt + 1), [
'pattern_id' => $pattern->id->toString(),
'channel' => $channelName,
'next_delay' => $nextDelay,
]);
} else {
$this->log('error', "All retry attempts exhausted for channel {$channelName}", [
'pattern_id' => $pattern->id->toString(),
'total_attempts' => $attempt,
]);
}
return false;
}
} catch (\Throwable $e) {
$this->log('error', "Retry failed with exception: {$e->getMessage()}", [
'pattern_id' => $pattern->id->toString(),
'channel' => $channelName,
'attempt' => $attempt,
'exception' => $e,
]);
return false;
}
}
/**
* Tests all configured channels
*/
public function testChannels(): array
{
$results = [];
foreach ($this->channels as $name => $channel) {
try {
$success = $channel->test();
$results[$name] = [
'status' => $success ? 'ok' : 'failed',
'test_time' => $this->clock->now()->format('c'),
];
$this->log('info', "Channel {$name} test: " . ($success ? 'OK' : 'FAILED'));
} catch (\Throwable $e) {
$results[$name] = [
'status' => 'error',
'error' => $e->getMessage(),
'test_time' => $this->clock->now()->format('c'),
];
$this->log('error', "Channel {$name} test error: {$e->getMessage()}");
}
}
return $results;
}
/**
* Gets alert statistics
*/
public function getStatistics(\DateTimeImmutable $from, \DateTimeImmutable $to): array
{
// This would typically query a database or metrics store
// For now, return basic info
return [
'period' => [
'from' => $from->format('c'),
'to' => $to->format('c'),
],
'channels' => array_keys($this->channels),
'urgency_mapping' => $this->channelsByUrgency,
];
}
private function isThrottled(ErrorPattern $pattern, AlertUrgency $urgency): bool
{
$throttleKey = self::THROTTLE_CACHE_PREFIX . $pattern->fingerprint;
$lastAlert = $this->cache->get($throttleKey);
if ($lastAlert === null) {
return false;
}
$throttleWindow = $this->getThrottleWindow($urgency);
$now = $this->clock->now()->getTimestamp();
return ($now - $lastAlert) < $throttleWindow;
}
private function updateThrottle(ErrorPattern $pattern, AlertUrgency $urgency): void
{
$throttleKey = self::THROTTLE_CACHE_PREFIX . $pattern->fingerprint;
$throttleWindow = $this->getThrottleWindow($urgency);
$this->cache->set($throttleKey, $this->clock->now()->getTimestamp(), $throttleWindow);
}
private function getThrottleWindow(AlertUrgency $urgency): int
{
return $this->throttleConfig[$urgency->value] ?? match ($urgency) {
AlertUrgency::URGENT => 300, // 5 minutes
AlertUrgency::HIGH => 900, // 15 minutes
AlertUrgency::MEDIUM => 3600, // 1 hour
AlertUrgency::LOW => 86400, // 24 hours
};
}
private function scheduleRetry(
ErrorPattern $pattern,
ErrorEvent $event,
string $channelName,
array $context,
int $attempt = 1,
?int $delay = null
): void {
$urgency = $pattern->getAlertUrgency();
$retryStrategy = $urgency->getRetryStrategy();
$delay ??= $retryStrategy['delays'][$attempt - 1] ?? 300;
$retryData = [
'pattern' => $pattern->toArray(),
'event' => $event->toArray(),
'channel' => $channelName,
'context' => $context,
'attempt' => $attempt,
'scheduled_at' => $this->clock->now()->format('c'),
];
$this->retryQueue->pushDelayed('alert_retries', $retryData, $delay);
}
private function scheduleEscalation(
ErrorPattern $pattern,
ErrorEvent $event,
array $failedChannels,
array $context
): void {
$escalationKey = self::ESCALATION_CACHE_PREFIX . $pattern->id->toString();
// Don't escalate if already escalated recently
if ($this->cache->get($escalationKey) !== null) {
return;
}
$urgency = $pattern->getAlertUrgency();
$escalationTimeout = $urgency->getEscalationTimeout();
$escalationData = [
'pattern' => $pattern->toArray(),
'event' => $event->toArray(),
'failed_channels' => $failedChannels,
'context' => $context,
'scheduled_at' => $this->clock->now()->format('c'),
];
$this->retryQueue->pushDelayed('alert_escalations', $escalationData, $escalationTimeout);
// Mark as escalated
$this->cache->set($escalationKey, true, $escalationTimeout * 2);
$this->log('warning', "Alert escalation scheduled", [
'pattern_id' => $pattern->id->toString(),
'failed_channels' => $failedChannels,
'escalation_timeout' => $escalationTimeout,
]);
}
private function log(string $level, string $message, array $context = []): void
{
if ($this->logger) {
match ($level) {
'debug' => $this->logger->debug("[AlertManager] {$message}", $context),
'info' => $this->logger->info("[AlertManager] {$message}", $context),
'warning' => $this->logger->warning("[AlertManager] {$message}", $context),
'error' => $this->logger->error("[AlertManager] {$message}", $context),
default => $this->logger->info("[AlertManager] {$message}", $context),
};
}
}
}

View File

@@ -0,0 +1,240 @@
<?php
declare(strict_types=1);
namespace App\Framework\ErrorAggregation\Alerting;
use App\Framework\ErrorAggregation\AlertUrgency;
use App\Framework\ErrorAggregation\ErrorEvent;
use App\Framework\ErrorAggregation\ErrorPattern;
use App\Framework\Mail\Message;
use App\Framework\Mail\Transport\TransportInterface;
/**
* Email alert channel implementation
*/
final readonly class EmailAlertChannel implements AlertChannel
{
private array $lastDeliveryStatus;
public function __construct(
private TransportInterface $transport,
private array $recipients = [],
private string $fromEmail = 'alerts@example.com',
private string $fromName = 'Error Alert System'
) {
$this->lastDeliveryStatus = ['status' => 'pending'];
}
public function getName(): string
{
return 'email';
}
public function sendAlert(ErrorPattern $pattern, ErrorEvent $triggeringEvent, array $context = []): bool
{
try {
$subject = $this->buildSubject($pattern, $triggeringEvent);
$body = $this->buildBody($pattern, $triggeringEvent, $context);
$message = new Message(
to: $this->recipients,
from: $this->fromEmail,
fromName: $this->fromName,
subject: $subject,
body: $body,
isHtml: true
);
$result = $this->transport->send($message);
$this->lastDeliveryStatus = [
'status' => $result ? 'sent' : 'failed',
'timestamp' => date('c'),
'pattern_id' => $pattern->id->toString(),
'recipients' => $this->recipients,
];
return $result;
} catch (\Throwable $e) {
$this->lastDeliveryStatus = [
'status' => 'error',
'timestamp' => date('c'),
'error' => $e->getMessage(),
'pattern_id' => $pattern->id->toString(),
];
return false;
}
}
public function canHandle(AlertUrgency $urgency): bool
{
// Email can handle all urgency levels
return true;
}
public function getLastDeliveryStatus(): array
{
return $this->lastDeliveryStatus;
}
public function test(): bool
{
try {
$testMessage = new Message(
to: $this->recipients,
from: $this->fromEmail,
fromName: $this->fromName,
subject: 'Alert System Test',
body: 'This is a test message from the error alert system.',
isHtml: false
);
return $this->transport->send($testMessage);
} catch (\Throwable) {
return false;
}
}
private function buildSubject(ErrorPattern $pattern, ErrorEvent $triggeringEvent): string
{
$urgency = $pattern->getAlertUrgency();
$urgencyPrefix = match ($urgency) {
AlertUrgency::URGENT => '[URGENT]',
AlertUrgency::HIGH => '[HIGH]',
AlertUrgency::MEDIUM => '[MEDIUM]',
AlertUrgency::LOW => '[LOW]',
};
$subject = "{$urgencyPrefix} Error Alert: {$pattern->service}";
if ($pattern->isCriticalPattern()) {
$subject = "[CRITICAL] {$subject}";
}
return $subject;
}
private function buildBody(ErrorPattern $pattern, ErrorEvent $triggeringEvent, array $context): string
{
$urgency = $pattern->getAlertUrgency();
$urgencyColor = match ($urgency) {
AlertUrgency::URGENT => '#dc2626',
AlertUrgency::HIGH => '#ea580c',
AlertUrgency::MEDIUM => '#ca8a04',
AlertUrgency::LOW => '#2563eb',
};
$html = "
<!DOCTYPE html>
<html>
<head>
<meta charset='UTF-8'>
<title>Error Alert</title>
<style>
body { font-family: Arial, sans-serif; line-height: 1.6; color: #333; }
.header { background: {$urgencyColor}; color: white; padding: 20px; border-radius: 5px 5px 0 0; }
.content { background: #f9f9f9; padding: 20px; border: 1px solid #ddd; border-top: none; }
.details { background: white; padding: 15px; margin: 10px 0; border-radius: 5px; border-left: 4px solid {$urgencyColor}; }
.metric { display: inline-block; margin-right: 20px; }
.metric strong { color: {$urgencyColor}; }
.footer { background: #333; color: white; padding: 10px; text-align: center; border-radius: 0 0 5px 5px; font-size: 12px; }
.critical-banner { background: #dc2626; color: white; padding: 10px; text-align: center; font-weight: bold; margin-bottom: 10px; }
</style>
</head>
<body>
";
if ($pattern->isCriticalPattern()) {
$html .= "<div class='critical-banner'>🚨 CRITICAL PATTERN DETECTED 🚨</div>";
}
$html .= "
<div class='header'>
<h2>{$pattern->severity->getIcon()} Error Alert - {$pattern->service}</h2>
<p>Urgency: <strong>{$urgency->value}</strong> | Pattern ID: {$pattern->id->toString()}</p>
</div>
<div class='content'>
<div class='details'>
<h3>Error Pattern Details</h3>
<p><strong>Service:</strong> {$pattern->service}</p>
<p><strong>Component:</strong> {$pattern->component}</p>
<p><strong>Operation:</strong> {$pattern->operation}</p>
<p><strong>Error Code:</strong> {$pattern->errorCode}</p>
<p><strong>Message:</strong> {$pattern->normalizedMessage}</p>
</div>
<div class='details'>
<h3>Occurrence Statistics</h3>
<div class='metric'><strong>{$pattern->occurrenceCount}</strong> Total Occurrences</div>
<div class='metric'><strong>" . number_format($pattern->getFrequency(), 2) . "</strong> Errors/min</div>
<div class='metric'><strong>" . count($pattern->affectedUsers) . "</strong> Affected Users</div>
<div class='metric'><strong>" . count($pattern->affectedIps) . "</strong> Affected IPs</div>
</div>
<div class='details'>
<h3>Timeline</h3>
<p><strong>First Seen:</strong> {$pattern->firstOccurrence->format('Y-m-d H:i:s T')}</p>
<p><strong>Last Seen:</strong> {$pattern->lastOccurrence->format('Y-m-d H:i:s T')}</p>
<p><strong>Duration:</strong> " . $this->formatDuration($pattern->firstOccurrence, $pattern->lastOccurrence) . "</p>
</div>
<div class='details'>
<h3>Triggering Event</h3>
<p><strong>Event ID:</strong> {$triggeringEvent->id->toString()}</p>
<p><strong>Request ID:</strong> {$triggeringEvent->requestId}</p>
<p><strong>User ID:</strong> " . ($triggeringEvent->userId ?? 'N/A') . "</p>
<p><strong>Client IP:</strong> {$triggeringEvent->clientIp}</p>
<p><strong>Occurred At:</strong> {$triggeringEvent->occurredAt->format('Y-m-d H:i:s T')}</p>
</div>
";
if ($triggeringEvent->isSecurityEvent) {
$html .= "
<div class='details' style='border-left-color: #dc2626; background-color: #fef2f2;'>
<h3>🛡️ Security Event</h3>
<p><strong>This is a security-related error that requires immediate attention.</strong></p>
</div>
";
}
if (! empty($context['dashboard_url'])) {
$html .= "
<div class='details'>
<h3>Actions</h3>
<p><a href='{$context['dashboard_url']}' style='background: {$urgencyColor}; color: white; padding: 10px 15px; text-decoration: none; border-radius: 3px;'>View in Dashboard</a></p>
</div>
";
}
$html .= "
</div>
<div class='footer'>
Error Alert System | Pattern Fingerprint: {$pattern->fingerprint}
</div>
</body>
</html>
";
return $html;
}
private function formatDuration(\DateTimeImmutable $start, \DateTimeImmutable $end): string
{
$diff = $end->getTimestamp() - $start->getTimestamp();
if ($diff < 60) {
return "{$diff} seconds";
} elseif ($diff < 3600) {
return round($diff / 60) . " minutes";
} elseif ($diff < 86400) {
return round($diff / 3600, 1) . " hours";
} else {
return round($diff / 86400, 1) . " days";
}
}
}