Michael Schiemer
fc3d7e6357
- Add comprehensive health check system with multiple endpoints - Add Prometheus metrics endpoint - Add production logging configurations (5 strategies) - Add complete deployment documentation suite: * QUICKSTART.md - 30-minute deployment guide * DEPLOYMENT_CHECKLIST.md - Printable verification checklist * DEPLOYMENT_WORKFLOW.md - Complete deployment lifecycle * PRODUCTION_DEPLOYMENT.md - Comprehensive technical reference * production-logging.md - Logging configuration guide * ANSIBLE_DEPLOYMENT.md - Infrastructure as Code automation * README.md - Navigation hub * DEPLOYMENT_SUMMARY.md - Executive summary - Add deployment scripts and automation - Add DEPLOYMENT_PLAN.md - Concrete plan for immediate deployment - Update README with production-ready features All production infrastructure is now complete and ready for deployment.
901 lines
37 KiB
PHP
901 lines
37 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use App\Framework\Core\ValueObjects\Byte;
|
|
use App\Framework\Core\ValueObjects\Duration;
|
|
use App\Framework\Core\ValueObjects\Percentage;
|
|
use App\Framework\Queue\Entities\Worker;
|
|
use App\Framework\Queue\Services\WorkerHealthCheckService;
|
|
use App\Framework\Queue\Services\WorkerRegistry;
|
|
use App\Framework\Queue\ValueObjects\QueueName;
|
|
use App\Framework\Queue\ValueObjects\WorkerId;
|
|
|
|
describe('Worker Management System', function () {
|
|
beforeEach(function () {
|
|
// Mock database connection for testing
|
|
$this->mockConnection = new class () {
|
|
private array $data = [];
|
|
|
|
private int $lastInsertId = 0;
|
|
|
|
private int $rowCount = 0;
|
|
|
|
public function prepare(string $sql): object
|
|
{
|
|
return new class ($sql, $this) {
|
|
public function __construct(
|
|
private string $sql,
|
|
private object $connection
|
|
) {
|
|
}
|
|
|
|
public function execute(array $params = []): bool
|
|
{
|
|
// Simulate different SQL operations
|
|
if (str_contains($this->sql, 'INSERT INTO queue_workers')) {
|
|
$this->connection->rowCount = 1;
|
|
$this->connection->data['workers'][$params['id']] = $params;
|
|
} elseif (str_contains($this->sql, 'UPDATE queue_workers')) {
|
|
if (str_contains($this->sql, 'is_active = 0')) {
|
|
// Deregister operation
|
|
if (isset($this->connection->data['workers'][$params['id']])) {
|
|
$this->connection->data['workers'][$params['id']]['is_active'] = 0;
|
|
$this->connection->rowCount = 1;
|
|
}
|
|
} else {
|
|
// Heartbeat update
|
|
if (isset($this->connection->data['workers'][$params['id']])) {
|
|
$worker = &$this->connection->data['workers'][$params['id']];
|
|
$worker['cpu_usage'] = $params['cpu_usage'];
|
|
$worker['memory_usage_bytes'] = $params['memory_usage_bytes'];
|
|
$worker['current_jobs'] = $params['current_jobs'];
|
|
$worker['last_heartbeat'] = date('Y-m-d H:i:s');
|
|
$this->connection->rowCount = 1;
|
|
}
|
|
}
|
|
} elseif (str_contains($this->sql, 'INSERT INTO worker_health_checks')) {
|
|
$this->connection->data['health_checks'][] = $params;
|
|
$this->connection->rowCount = 1;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public function fetch(): array|false
|
|
{
|
|
if (str_contains($this->sql, 'SELECT * FROM queue_workers WHERE id = :id')) {
|
|
$id = func_get_args()[0]['id'] ?? null;
|
|
|
|
return $this->connection->data['workers'][$id] ?? false;
|
|
}
|
|
|
|
if (str_contains($this->sql, 'SELECT * FROM queue_workers') && str_contains($this->sql, 'is_active = 1')) {
|
|
// Return first active worker for testing
|
|
foreach ($this->connection->data['workers'] ?? [] as $worker) {
|
|
if ($worker['is_active']) {
|
|
return $worker;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
// Statistics query
|
|
if (str_contains($this->sql, 'COUNT(*) as total_workers')) {
|
|
return [
|
|
'total_workers' => count($this->connection->data['workers'] ?? []),
|
|
'active_workers' => count(array_filter($this->connection->data['workers'] ?? [], fn ($w) => $w['is_active'])),
|
|
'healthy_workers' => count(array_filter($this->connection->data['workers'] ?? [], fn ($w) => $w['is_active'])),
|
|
'total_capacity' => 100,
|
|
'current_load' => 50,
|
|
'avg_cpu_usage' => 25.5,
|
|
'avg_memory_usage' => 1024 * 1024 * 512, // 512MB
|
|
'unique_hosts' => 2,
|
|
];
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
public function fetchAll(): array
|
|
{
|
|
if (str_contains($this->sql, 'worker_health_checks') && str_contains($this->sql, 'GROUP BY')) {
|
|
return [
|
|
[
|
|
'check_date' => '2024-01-01',
|
|
'check_hour' => '14',
|
|
'avg_score' => 85.5,
|
|
'total_checks' => 10,
|
|
'healthy_count' => 8,
|
|
'warning_count' => 2,
|
|
'critical_count' => 0,
|
|
],
|
|
];
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
public function rowCount(): int
|
|
{
|
|
return $this->connection->rowCount;
|
|
}
|
|
};
|
|
}
|
|
|
|
public function setWorkerData(array $workers): void
|
|
{
|
|
$this->data['workers'] = $workers;
|
|
}
|
|
};
|
|
|
|
// Mock logger for testing
|
|
$this->mockLogger = new class () {
|
|
public array $logs = [];
|
|
|
|
public function info(string $message, array $context = []): void
|
|
{
|
|
$this->logs[] = ['level' => 'info', 'message' => $message, 'context' => $context];
|
|
}
|
|
|
|
public function debug(string $message, array $context = []): void
|
|
{
|
|
$this->logs[] = ['level' => 'debug', 'message' => $message, 'context' => $context];
|
|
}
|
|
|
|
public function error(string $message, array $context = []): void
|
|
{
|
|
$this->logs[] = ['level' => 'error', 'message' => $message, 'context' => $context];
|
|
}
|
|
|
|
public function warning(string $message, array $context = []): void
|
|
{
|
|
$this->logs[] = ['level' => 'warning', 'message' => $message, 'context' => $context];
|
|
}
|
|
};
|
|
|
|
$this->workerRegistry = new WorkerRegistry($this->mockConnection, $this->mockLogger);
|
|
$this->healthCheckService = new WorkerHealthCheckService($this->workerRegistry, $this->mockConnection, $this->mockLogger);
|
|
});
|
|
|
|
describe('WorkerId Value Object', function () {
|
|
it('can generate unique worker IDs', function () {
|
|
$id1 = WorkerId::generate();
|
|
$id2 = WorkerId::generate();
|
|
|
|
expect($id1->toString())->not()->toBe($id2->toString());
|
|
expect($id1->getValue())->not()->toBeEmpty();
|
|
});
|
|
|
|
it('can create deterministic IDs for host and process', function () {
|
|
$id1 = WorkerId::forHost('localhost', 1234);
|
|
$id2 = WorkerId::forHost('localhost', 1234);
|
|
|
|
expect($id1->toString())->toBe($id2->toString());
|
|
expect($id1->getValue())->toHaveLength(16);
|
|
});
|
|
|
|
it('can create from string and validate', function () {
|
|
$original = 'worker_123_test';
|
|
$id = WorkerId::fromString($original);
|
|
|
|
expect($id->toString())->toBe($original);
|
|
expect($id->getValue())->toBe($original);
|
|
expect((string) $id)->toBe($original);
|
|
});
|
|
|
|
it('throws exception for empty worker ID', function () {
|
|
expect(fn () => WorkerId::fromString(''))
|
|
->toThrow(\InvalidArgumentException::class, 'WorkerId cannot be empty');
|
|
});
|
|
|
|
it('can compare worker IDs for equality', function () {
|
|
$id1 = WorkerId::fromString('worker_123');
|
|
$id2 = WorkerId::fromString('worker_123');
|
|
$id3 = WorkerId::fromString('worker_456');
|
|
|
|
expect($id1->equals($id2))->toBeTrue();
|
|
expect($id1->equals($id3))->toBeFalse();
|
|
});
|
|
|
|
it('supports JSON serialization', function () {
|
|
$id = WorkerId::fromString('worker_test_123');
|
|
$json = json_encode($id);
|
|
|
|
expect($json)->toBe('"worker_test_123"');
|
|
});
|
|
});
|
|
|
|
describe('Worker Entity', function () {
|
|
it('can register a new worker with valid parameters', function () {
|
|
$queues = [QueueName::default(), QueueName::high()];
|
|
$capabilities = ['pdf_processing', 'email_sending'];
|
|
|
|
$worker = Worker::register(
|
|
hostname: 'worker-01',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
capabilities: $capabilities
|
|
);
|
|
|
|
expect($worker->hostname)->toBe('worker-01');
|
|
expect($worker->processId)->toBe(1234);
|
|
expect($worker->queues)->toBe($queues);
|
|
expect($worker->maxJobs)->toBe(5);
|
|
expect($worker->isActive)->toBeTrue();
|
|
expect($worker->capabilities)->toBe($capabilities);
|
|
expect($worker->currentJobs)->toBe(0);
|
|
expect($worker->version)->toBe('1.0.0');
|
|
expect($worker->registeredAt)->toBeInstanceOf(\DateTimeImmutable::class);
|
|
expect($worker->lastHeartbeat)->toBeInstanceOf(\DateTimeImmutable::class);
|
|
});
|
|
|
|
it('throws exception when no queues provided', function () {
|
|
expect(fn () => new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'test',
|
|
processId: 1234,
|
|
queues: [], // Empty queues
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable()
|
|
))->toThrow(\InvalidArgumentException::class, 'Worker must handle at least one queue');
|
|
});
|
|
|
|
it('validates job constraints during construction', function () {
|
|
$baseWorker = fn ($maxJobs, $currentJobs) => new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'test',
|
|
processId: 1234,
|
|
queues: [QueueName::default()],
|
|
maxJobs: $maxJobs,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
currentJobs: $currentJobs
|
|
);
|
|
|
|
// Invalid max jobs
|
|
expect(fn () => $baseWorker(0, 0))
|
|
->toThrow(\InvalidArgumentException::class, 'Max jobs must be greater than 0');
|
|
expect(fn () => $baseWorker(-1, 0))
|
|
->toThrow(\InvalidArgumentException::class, 'Max jobs must be greater than 0');
|
|
|
|
// Invalid current jobs
|
|
expect(fn () => $baseWorker(5, -1))
|
|
->toThrow(\InvalidArgumentException::class, 'Current jobs cannot be negative');
|
|
expect(fn () => $baseWorker(5, 10))
|
|
->toThrow(\InvalidArgumentException::class, 'Current jobs cannot exceed max jobs');
|
|
});
|
|
|
|
it('can update heartbeat with new metrics', function () {
|
|
$worker = Worker::register('worker-01', 1234, [QueueName::default()]);
|
|
$newCpuUsage = new Percentage(75.5);
|
|
$newMemoryUsage = Byte::fromMegabytes(512);
|
|
$newCurrentJobs = 3;
|
|
|
|
$updatedWorker = $worker->updateHeartbeat($newCpuUsage, $newMemoryUsage, $newCurrentJobs);
|
|
|
|
expect($updatedWorker->cpuUsage)->toBe($newCpuUsage);
|
|
expect($updatedWorker->memoryUsage)->toBe($newMemoryUsage);
|
|
expect($updatedWorker->currentJobs)->toBe($newCurrentJobs);
|
|
expect($updatedWorker->isActive)->toBeTrue();
|
|
expect($updatedWorker->lastHeartbeat->getTimestamp())->toBeGreaterThan($worker->lastHeartbeat->getTimestamp());
|
|
});
|
|
|
|
it('can mark worker as inactive', function () {
|
|
$worker = Worker::register('worker-01', 1234, [QueueName::default()]);
|
|
$inactiveWorker = $worker->markInactive();
|
|
|
|
expect($inactiveWorker->isActive)->toBeFalse();
|
|
expect($inactiveWorker->id)->toBe($worker->id);
|
|
expect($inactiveWorker->hostname)->toBe($worker->hostname);
|
|
});
|
|
|
|
it('correctly determines worker availability for jobs', function () {
|
|
$queues = [QueueName::default()];
|
|
|
|
// Healthy active worker with capacity
|
|
$healthyWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'healthy-worker',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable(),
|
|
isActive: true,
|
|
cpuUsage: new Percentage(50),
|
|
memoryUsage: Byte::fromMegabytes(500),
|
|
currentJobs: 2
|
|
);
|
|
|
|
expect($healthyWorker->isAvailableForJobs())->toBeTrue();
|
|
|
|
// Inactive worker
|
|
$inactiveWorker = $healthyWorker->markInactive();
|
|
expect($inactiveWorker->isAvailableForJobs())->toBeFalse();
|
|
|
|
// Worker at max capacity
|
|
$maxCapacityWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'maxed-worker',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable(),
|
|
isActive: true,
|
|
currentJobs: 5 // At max capacity
|
|
);
|
|
|
|
expect($maxCapacityWorker->isAvailableForJobs())->toBeFalse();
|
|
|
|
// Unhealthy worker (high CPU)
|
|
$unhealthyWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'unhealthy-worker',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable(),
|
|
isActive: true,
|
|
cpuUsage: new Percentage(95), // Too high
|
|
currentJobs: 2
|
|
);
|
|
|
|
expect($unhealthyWorker->isAvailableForJobs())->toBeFalse();
|
|
});
|
|
|
|
it('can check queue handling capability', function () {
|
|
$defaultQueue = QueueName::default();
|
|
$highQueue = QueueName::high();
|
|
$lowQueue = QueueName::low();
|
|
|
|
$worker = Worker::register('worker-01', 1234, [$defaultQueue, $highQueue]);
|
|
|
|
expect($worker->handlesQueue($defaultQueue))->toBeTrue();
|
|
expect($worker->handlesQueue($highQueue))->toBeTrue();
|
|
expect($worker->handlesQueue($lowQueue))->toBeFalse();
|
|
});
|
|
|
|
it('determines health status based on multiple factors', function () {
|
|
$queues = [QueueName::default()];
|
|
|
|
// Healthy worker
|
|
$healthyWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'healthy',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable(), // Recent heartbeat
|
|
isActive: true,
|
|
cpuUsage: new Percentage(50),
|
|
memoryUsage: Byte::fromMegabytes(500) // 500MB < 2GB limit
|
|
);
|
|
|
|
expect($healthyWorker->isHealthy())->toBeTrue();
|
|
|
|
// Worker with old heartbeat
|
|
$staleWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'stale',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable('-2 minutes'), // Too old
|
|
isActive: true
|
|
);
|
|
|
|
expect($staleWorker->isHealthy())->toBeFalse();
|
|
|
|
// Worker with high CPU
|
|
$highCpuWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'high-cpu',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable(),
|
|
isActive: true,
|
|
cpuUsage: new Percentage(95) // Too high
|
|
);
|
|
|
|
expect($highCpuWorker->isHealthy())->toBeFalse();
|
|
|
|
// Worker with high memory
|
|
$highMemoryWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'high-memory',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable(),
|
|
isActive: true,
|
|
memoryUsage: Byte::fromGigabytes(3) // > 2GB limit
|
|
);
|
|
|
|
expect($highMemoryWorker->isHealthy())->toBeFalse();
|
|
});
|
|
|
|
it('calculates load percentage correctly', function () {
|
|
$queues = [QueueName::default()];
|
|
|
|
// Job load higher than CPU load
|
|
$jobLoadWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'job-load',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 10,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
currentJobs: 8, // 80% job load
|
|
cpuUsage: new Percentage(30) // 30% CPU load
|
|
);
|
|
|
|
expect($jobLoadWorker->getLoadPercentage()->getValue())->toBe(80.0);
|
|
|
|
// CPU load higher than job load
|
|
$cpuLoadWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'cpu-load',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 10,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
currentJobs: 3, // 30% job load
|
|
cpuUsage: new Percentage(70) // 70% CPU load
|
|
);
|
|
|
|
expect($cpuLoadWorker->getLoadPercentage()->getValue())->toBe(70.0);
|
|
|
|
// Zero max jobs edge case
|
|
$zeroMaxWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'zero-max',
|
|
processId: 1234,
|
|
queues: $queues,
|
|
maxJobs: 1,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
currentJobs: 1 // 100% job load
|
|
);
|
|
|
|
expect($zeroMaxWorker->getLoadPercentage()->getValue())->toBe(100.0);
|
|
});
|
|
|
|
it('can check capabilities', function () {
|
|
$worker = Worker::register('worker-01', 1234, [QueueName::default()], 5, ['pdf', 'email', 'resize']);
|
|
|
|
expect($worker->hasCapability('pdf'))->toBeTrue();
|
|
expect($worker->hasCapability('email'))->toBeTrue();
|
|
expect($worker->hasCapability('video'))->toBeFalse();
|
|
});
|
|
|
|
it('can convert to monitoring array format', function () {
|
|
$worker = Worker::register('worker-01', 1234, [QueueName::default()], 5, ['pdf']);
|
|
$monitoring = $worker->toMonitoringArray();
|
|
|
|
expect($monitoring)->toHaveKey('id');
|
|
expect($monitoring)->toHaveKey('hostname');
|
|
expect($monitoring)->toHaveKey('process_id');
|
|
expect($monitoring)->toHaveKey('queues');
|
|
expect($monitoring)->toHaveKey('is_healthy');
|
|
expect($monitoring)->toHaveKey('is_available');
|
|
expect($monitoring)->toHaveKey('load_percentage');
|
|
expect($monitoring)->toHaveKey('capabilities');
|
|
|
|
expect($monitoring['hostname'])->toBe('worker-01');
|
|
expect($monitoring['process_id'])->toBe(1234);
|
|
expect($monitoring['capabilities'])->toBe(['pdf']);
|
|
});
|
|
|
|
it('can serialize to and from array', function () {
|
|
$queues = [QueueName::default(), QueueName::high()];
|
|
$capabilities = ['pdf', 'email'];
|
|
|
|
$worker = Worker::register('worker-01', 1234, $queues, 5, $capabilities);
|
|
$array = $worker->toArray();
|
|
|
|
expect($array)->toHaveKey('id');
|
|
expect($array)->toHaveKey('hostname');
|
|
expect($array)->toHaveKey('queues');
|
|
expect($array)->toHaveKey('capabilities');
|
|
|
|
// Test that queues and capabilities are JSON encoded
|
|
expect($array['queues'])->toBeString();
|
|
expect($array['capabilities'])->toBeString();
|
|
|
|
// Note: fromArray() is simplified in the implementation
|
|
// In real testing, you'd want to test full serialization/deserialization
|
|
$restoredWorker = Worker::fromArray($array);
|
|
expect($restoredWorker->hostname)->toBe('worker-01');
|
|
expect($restoredWorker->processId)->toBe(1234);
|
|
});
|
|
});
|
|
|
|
describe('WorkerRegistry Service', function () {
|
|
it('can register a worker successfully', function () {
|
|
$worker = Worker::register('test-host', 1234, [QueueName::default()]);
|
|
|
|
$this->workerRegistry->register($worker);
|
|
|
|
expect($this->mockLogger->logs)->toContain([
|
|
'level' => 'info',
|
|
'message' => 'Registering worker',
|
|
'context' => [
|
|
'worker_id' => $worker->id->toString(),
|
|
'hostname' => 'test-host',
|
|
'process_id' => 1234,
|
|
'queues' => [QueueName::default()],
|
|
'max_jobs' => 10,
|
|
],
|
|
]);
|
|
|
|
expect($this->mockLogger->logs)->toContain([
|
|
'level' => 'debug',
|
|
'message' => 'Worker registered successfully',
|
|
'context' => ['worker_id' => $worker->id->toString()],
|
|
]);
|
|
});
|
|
|
|
it('can deregister a worker', function () {
|
|
$workerId = WorkerId::generate();
|
|
|
|
// Setup worker data in mock
|
|
$this->mockConnection->setWorkerData([
|
|
$workerId->toString() => [
|
|
'id' => $workerId->toString(),
|
|
'is_active' => 1,
|
|
],
|
|
]);
|
|
|
|
$this->workerRegistry->deregister($workerId);
|
|
|
|
expect($this->mockLogger->logs)->toContain([
|
|
'level' => 'info',
|
|
'message' => 'Deregistering worker',
|
|
'context' => ['worker_id' => $workerId->toString()],
|
|
]);
|
|
});
|
|
|
|
it('can update worker heartbeat', function () {
|
|
$workerId = WorkerId::generate();
|
|
$cpuUsage = new Percentage(45.5);
|
|
$memoryUsage = Byte::fromMegabytes(256);
|
|
$currentJobs = 3;
|
|
|
|
// Setup worker data in mock
|
|
$this->mockConnection->setWorkerData([
|
|
$workerId->toString() => [
|
|
'id' => $workerId->toString(),
|
|
'is_active' => 1,
|
|
],
|
|
]);
|
|
|
|
$this->workerRegistry->updateHeartbeat($workerId, $cpuUsage, $memoryUsage, $currentJobs);
|
|
|
|
// Should not log warnings when worker found and updated
|
|
$warningLogs = array_filter($this->mockLogger->logs, fn ($log) => $log['level'] === 'warning');
|
|
expect($warningLogs)->toBeEmpty();
|
|
});
|
|
|
|
it('logs warning when heartbeat update fails for non-existent worker', function () {
|
|
$workerId = WorkerId::generate();
|
|
$cpuUsage = new Percentage(50);
|
|
$memoryUsage = Byte::fromMegabytes(512);
|
|
|
|
// No worker data setup - worker doesn't exist
|
|
|
|
$this->workerRegistry->updateHeartbeat($workerId, $cpuUsage, $memoryUsage, 2);
|
|
|
|
$warningLogs = array_filter($this->mockLogger->logs, fn ($log) => $log['level'] === 'warning');
|
|
expect($warningLogs)->not()->toBeEmpty();
|
|
});
|
|
|
|
it('can find worker by ID', function () {
|
|
$workerId = WorkerId::generate();
|
|
$workerData = [
|
|
'id' => $workerId->toString(),
|
|
'hostname' => 'test-worker',
|
|
'process_id' => 1234,
|
|
'queues' => json_encode(['default']),
|
|
'max_jobs' => 5,
|
|
'current_jobs' => 2,
|
|
'is_active' => 1,
|
|
'cpu_usage' => 50,
|
|
'memory_usage_bytes' => 512 * 1024 * 1024,
|
|
'registered_at' => '2024-01-01 12:00:00',
|
|
'last_heartbeat' => '2024-01-01 12:05:00',
|
|
'capabilities' => json_encode(['pdf']),
|
|
'version' => '1.0.0',
|
|
];
|
|
|
|
$this->mockConnection->setWorkerData([$workerId->toString() => $workerData]);
|
|
|
|
$foundWorker = $this->workerRegistry->findById($workerId);
|
|
|
|
expect($foundWorker)->toBeInstanceOf(Worker::class);
|
|
expect($foundWorker->hostname)->toBe('test-worker');
|
|
expect($foundWorker->processId)->toBe(1234);
|
|
});
|
|
|
|
it('returns null when worker not found by ID', function () {
|
|
$workerId = WorkerId::generate();
|
|
|
|
$foundWorker = $this->workerRegistry->findById($workerId);
|
|
|
|
expect($foundWorker)->toBeNull();
|
|
});
|
|
|
|
it('can get worker statistics', function () {
|
|
$stats = $this->workerRegistry->getWorkerStatistics();
|
|
|
|
expect($stats)->toHaveKey('total_workers');
|
|
expect($stats)->toHaveKey('active_workers');
|
|
expect($stats)->toHaveKey('healthy_workers');
|
|
expect($stats)->toHaveKey('total_capacity');
|
|
expect($stats)->toHaveKey('current_load');
|
|
expect($stats)->toHaveKey('capacity_utilization');
|
|
expect($stats)->toHaveKey('avg_cpu_usage');
|
|
expect($stats)->toHaveKey('avg_memory_usage_mb');
|
|
expect($stats)->toHaveKey('unique_hosts');
|
|
expect($stats)->toHaveKey('queue_distribution');
|
|
|
|
expect($stats['capacity_utilization'])->toBe(50.0);
|
|
expect($stats['avg_memory_usage_mb'])->toBe(512.0);
|
|
});
|
|
|
|
it('can cleanup inactive workers', function () {
|
|
$cleanedCount = $this->workerRegistry->cleanupInactiveWorkers(5);
|
|
|
|
expect($cleanedCount)->toBeInt();
|
|
|
|
expect($this->mockLogger->logs)->toContain([
|
|
'level' => 'info',
|
|
'message' => 'Starting cleanup of inactive workers',
|
|
'context' => ['inactive_minutes' => 5],
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('WorkerHealthCheckService', function () {
|
|
it('can perform health check on individual worker', function () {
|
|
$worker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'test-worker',
|
|
processId: 1234,
|
|
queues: [QueueName::default()],
|
|
maxJobs: 10,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable(),
|
|
isActive: true,
|
|
cpuUsage: new Percentage(45),
|
|
memoryUsage: Byte::fromMegabytes(800),
|
|
currentJobs: 5
|
|
);
|
|
|
|
$health = $this->healthCheckService->checkWorkerHealth($worker);
|
|
|
|
expect($health)->toHaveKey('worker_id');
|
|
expect($health)->toHaveKey('hostname');
|
|
expect($health)->toHaveKey('status');
|
|
expect($health)->toHaveKey('score');
|
|
expect($health)->toHaveKey('metrics');
|
|
expect($health)->toHaveKey('issues');
|
|
expect($health)->toHaveKey('warnings');
|
|
expect($health)->toHaveKey('checked_at');
|
|
|
|
expect($health['worker_id'])->toBe($worker->id->toString());
|
|
expect($health['hostname'])->toBe('test-worker');
|
|
expect($health['status'])->toBe('healthy');
|
|
expect($health['score'])->toBeGreaterThanOrEqual(80);
|
|
expect($health['issues'])->toBeEmpty();
|
|
});
|
|
|
|
it('detects critical health issues', function () {
|
|
$criticalWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'critical-worker',
|
|
processId: 1234,
|
|
queues: [QueueName::default()],
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable('-5 minutes'), // Very old heartbeat
|
|
isActive: true,
|
|
cpuUsage: new Percentage(95), // Critical CPU
|
|
memoryUsage: Byte::fromGigabytes(3), // Critical memory
|
|
currentJobs: 5 // Overloaded
|
|
);
|
|
|
|
$health = $this->healthCheckService->checkWorkerHealth($criticalWorker);
|
|
|
|
expect($health['status'])->toBe('critical');
|
|
expect($health['score'])->toBeLessThan(50);
|
|
expect($health['issues'])->not()->toBeEmpty();
|
|
expect($health['issues'])->toContain('No heartbeat for 300 seconds');
|
|
expect($health['issues'])->toContain('Critical CPU usage: 95%');
|
|
expect($health['issues'])->toContain('Critical memory usage: 3.000GB');
|
|
expect($health['issues'])->toContain('Worker overloaded: 95%');
|
|
});
|
|
|
|
it('detects warning conditions', function () {
|
|
$warningWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'warning-worker',
|
|
processId: 1234,
|
|
queues: [QueueName::default()],
|
|
maxJobs: 10,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
lastHeartbeat: new \DateTimeImmutable('-70 seconds'), // Slightly delayed
|
|
isActive: true,
|
|
cpuUsage: new Percentage(80), // Warning level CPU
|
|
memoryUsage: Byte::fromGigabytes(1.8), // Warning level memory
|
|
currentJobs: 9 // High load
|
|
);
|
|
|
|
$health = $this->healthCheckService->checkWorkerHealth($warningWorker);
|
|
|
|
expect($health['status'])->toBe('warning');
|
|
expect($health['score'])->toBeGreaterThan(30);
|
|
expect($health['score'])->toBeLessThan(80);
|
|
expect($health['warnings'])->not()->toBeEmpty();
|
|
expect($health['warnings'])->toContain('Heartbeat delayed (70s)');
|
|
expect($health['warnings'])->toContain('High CPU usage: 80%');
|
|
expect($health['warnings'])->toContain('High memory usage: 1.800GB');
|
|
expect($health['warnings'])->toContain('High worker load: 90%');
|
|
});
|
|
|
|
it('handles inactive workers correctly', function () {
|
|
$inactiveWorker = new Worker(
|
|
id: WorkerId::generate(),
|
|
hostname: 'inactive-worker',
|
|
processId: 1234,
|
|
queues: [QueueName::default()],
|
|
maxJobs: 5,
|
|
registeredAt: new \DateTimeImmutable(),
|
|
isActive: false // Inactive
|
|
);
|
|
|
|
$health = $this->healthCheckService->checkWorkerHealth($inactiveWorker);
|
|
|
|
expect($health['status'])->toBe('critical');
|
|
expect($health['score'])->toBe(0);
|
|
expect($health['issues'])->toContain('Worker marked as inactive');
|
|
});
|
|
|
|
it('can generate system health report', function () {
|
|
// Setup some test workers in the mock
|
|
$this->mockConnection->setWorkerData([
|
|
'worker1' => [
|
|
'id' => 'worker1',
|
|
'hostname' => 'test1',
|
|
'process_id' => 1234,
|
|
'queues' => json_encode(['default']),
|
|
'max_jobs' => 5,
|
|
'current_jobs' => 2,
|
|
'is_active' => 1,
|
|
'cpu_usage' => 50,
|
|
'memory_usage_bytes' => 512 * 1024 * 1024,
|
|
'registered_at' => '2024-01-01 12:00:00',
|
|
'last_heartbeat' => date('Y-m-d H:i:s'),
|
|
'capabilities' => json_encode([]),
|
|
'version' => '1.0.0',
|
|
],
|
|
]);
|
|
|
|
$report = $this->healthCheckService->generateSystemHealthReport();
|
|
|
|
expect($report)->toHaveKey('current_health');
|
|
expect($report)->toHaveKey('trends_24h');
|
|
expect($report)->toHaveKey('top_issues_24h');
|
|
expect($report)->toHaveKey('generated_at');
|
|
|
|
expect($report['current_health'])->toHaveKey('workers');
|
|
expect($report['current_health'])->toHaveKey('overall');
|
|
});
|
|
|
|
it('can cleanup old health check records', function () {
|
|
$deletedCount = $this->healthCheckService->cleanupHealthChecks(Duration::fromDays(7));
|
|
|
|
expect($deletedCount)->toBeInt();
|
|
|
|
expect($this->mockLogger->logs)->toContain([
|
|
'level' => 'info',
|
|
'message' => 'Health check cleanup completed',
|
|
'context' => [
|
|
'deleted_records' => $deletedCount,
|
|
'retention_days' => 7.0,
|
|
],
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('Integration Testing', function () {
|
|
it('can perform full worker lifecycle management', function () {
|
|
// 1. Register worker
|
|
$worker = Worker::register('integration-host', 9999, [QueueName::default(), QueueName::high()], 8, ['pdf']);
|
|
$this->workerRegistry->register($worker);
|
|
|
|
// 2. Update heartbeat
|
|
$this->workerRegistry->updateHeartbeat(
|
|
$worker->id,
|
|
new Percentage(65),
|
|
Byte::fromMegabytes(1024),
|
|
3
|
|
);
|
|
|
|
// 3. Perform health check
|
|
$health = $this->healthCheckService->checkWorkerHealth($worker);
|
|
expect($health['status'])->toBeIn(['healthy', 'warning']);
|
|
|
|
// 4. Find worker
|
|
$foundWorker = $this->workerRegistry->findById($worker->id);
|
|
expect($foundWorker)->not()->toBeNull();
|
|
|
|
// 5. Get statistics
|
|
$stats = $this->workerRegistry->getWorkerStatistics();
|
|
expect($stats['total_workers'])->toBeGreaterThanOrEqual(1);
|
|
|
|
// 6. Deregister worker
|
|
$this->workerRegistry->deregister($worker->id);
|
|
|
|
// Verify all operations logged appropriately
|
|
$logCount = count($this->mockLogger->logs);
|
|
expect($logCount)->toBeGreaterThan(0);
|
|
});
|
|
|
|
it('handles edge cases and error conditions gracefully', function () {
|
|
// Test with non-existent worker ID
|
|
$nonExistentId = WorkerId::generate();
|
|
$foundWorker = $this->workerRegistry->findById($nonExistentId);
|
|
expect($foundWorker)->toBeNull();
|
|
|
|
// Test heartbeat update for non-existent worker
|
|
$this->workerRegistry->updateHeartbeat(
|
|
$nonExistentId,
|
|
new Percentage(50),
|
|
Byte::fromMegabytes(512),
|
|
1
|
|
);
|
|
|
|
// Should log warning
|
|
$warningLogs = array_filter($this->mockLogger->logs, fn ($log) => $log['level'] === 'warning');
|
|
expect($warningLogs)->not()->toBeEmpty();
|
|
});
|
|
|
|
it('maintains data consistency across operations', function () {
|
|
$worker = Worker::register('consistency-test', 7777, [QueueName::default()]);
|
|
|
|
// Test immutability - operations should return new instances
|
|
$originalId = $worker->id;
|
|
$originalHostname = $worker->hostname;
|
|
|
|
$updatedWorker = $worker->updateHeartbeat(
|
|
new Percentage(30),
|
|
Byte::fromMegabytes(256),
|
|
1
|
|
);
|
|
|
|
// Original worker unchanged
|
|
expect($worker->id)->toBe($originalId);
|
|
expect($worker->hostname)->toBe($originalHostname);
|
|
expect($worker->currentJobs)->toBe(0);
|
|
|
|
// Updated worker has new values
|
|
expect($updatedWorker->id)->toBe($originalId); // Same ID
|
|
expect($updatedWorker->hostname)->toBe($originalHostname); // Same hostname
|
|
expect($updatedWorker->currentJobs)->toBe(1); // Updated jobs
|
|
expect($updatedWorker->cpuUsage->getValue())->toBe(30.0);
|
|
|
|
$inactiveWorker = $updatedWorker->markInactive();
|
|
expect($updatedWorker->isActive)->toBeTrue(); // Original still active
|
|
expect($inactiveWorker->isActive)->toBeFalse(); // New instance inactive
|
|
});
|
|
});
|
|
});
|