connection = mock(ConnectionInterface::class); $this->logger = mock(Logger::class); // Setup services $this->workerRegistry = new WorkerRegistry($this->connection, $this->logger); $this->distributedLock = new DatabaseDistributedLock($this->connection, $this->logger); $this->jobDistribution = new JobDistributionService( $this->workerRegistry, $this->distributedLock, $this->connection, $this->logger ); // Mock default logger behavior $this->logger->shouldReceive('info')->andReturn(null); $this->logger->shouldReceive('debug')->andReturn(null); $this->logger->shouldReceive('warning')->andReturn(null); $this->logger->shouldReceive('error')->andReturn(null); }); describe('E-commerce Order Processing Scenario', function () { it('handles peak shopping season with multiple worker types', function () { // Scenario: Black Friday traffic with specialized workers // Create specialized workers for different tasks $emailWorkers = [ Worker::register( hostname: 'email-server-1', processId: 2001, queues: [QueueName::emailQueue()], maxJobs: 20, capabilities: ['email', 'newsletter', 'notifications'] ), Worker::register( hostname: 'email-server-2', processId: 2002, queues: [QueueName::emailQueue()], maxJobs: 20, capabilities: ['email', 'newsletter', 'notifications'] ), ]; $imageWorkers = [ Worker::register( hostname: 'image-server-1', processId: 3001, queues: [QueueName::fromString('image-processing')], maxJobs: 5, // Resource intensive capabilities: ['image-resize', 'thumbnail', 'watermark'] ), ]; $generalWorkers = [ Worker::register( hostname: 'app-server-1', processId: 1001, queues: [ QueueName::defaultQueue(), QueueName::fromString('reports'), ], maxJobs: 15, capabilities: ['pdf-generation', 'reporting', 'exports'] ), Worker::register( hostname: 'app-server-2', processId: 1002, queues: [ QueueName::defaultQueue(), QueueName::fromString('reports'), ], maxJobs: 15, capabilities: ['pdf-generation', 'reporting', 'exports'] ), ]; $allWorkers = array_merge($emailWorkers, $imageWorkers, $generalWorkers); // Mock worker registration foreach ($allWorkers as $worker) { $stmt = mock(\PDOStatement::class); $stmt->shouldReceive('execute')->andReturn(true); $this->connection->shouldReceive('prepare')->andReturn($stmt); $this->workerRegistry->register($worker); } // Simulate different job types being distributed $jobs = [ // Email jobs (high volume, low resource) ['id' => JobId::generate(), 'queue' => QueueName::emailQueue(), 'type' => 'order-confirmation'], ['id' => JobId::generate(), 'queue' => QueueName::emailQueue(), 'type' => 'shipping-notification'], ['id' => JobId::generate(), 'queue' => QueueName::emailQueue(), 'type' => 'newsletter'], // Image processing jobs (low volume, high resource) ['id' => JobId::generate(), 'queue' => QueueName::fromString('image-processing'), 'type' => 'product-thumbnails'], // General processing jobs ['id' => JobId::generate(), 'queue' => QueueName::defaultQueue(), 'type' => 'invoice-generation'], ['id' => JobId::generate(), 'queue' => QueueName::fromString('reports'), 'type' => 'sales-report'], ]; // Mock job distribution foreach ($jobs as $job) { // Mock finding workers for queue $workerStmt = mock(\PDOStatement::class); $workerStmt->shouldReceive('execute')->andReturn(true); $workerStmt->shouldReceive('fetch')->andReturn(false); // Simplified for test // Mock lock operations $lockStmt = mock(\PDOStatement::class); $lockStmt->shouldReceive('execute')->andReturn(true); $lockStmt->shouldReceive('rowCount')->andReturn(1); $this->connection->shouldReceive('prepare')->andReturn($workerStmt, $lockStmt); $assignedWorker = $this->jobDistribution->findBestWorkerForJob($job['queue']); // In real scenario, would validate worker assignment logic } // Verify system can handle the load expect(count($allWorkers))->toBe(5); expect(count($jobs))->toBe(6); }); it('handles worker failure during peak traffic gracefully', function () { // Scenario: Worker crashes during high load, jobs need redistribution $healthyWorker = Worker::register( hostname: 'stable-server', processId: 1001, queues: [QueueName::defaultQueue()], maxJobs: 10, capabilities: ['email', 'pdf-generation'] ); $failingWorker = Worker::register( hostname: 'failing-server', processId: 1002, queues: [QueueName::defaultQueue()], maxJobs: 10, capabilities: ['email', 'pdf-generation'] ); // Simulate worker failure (stale heartbeat, high resource usage) $failedWorker = new Worker( id: $failingWorker->id, hostname: $failingWorker->hostname, processId: $failingWorker->processId, queues: $failingWorker->queues, maxJobs: $failingWorker->maxJobs, registeredAt: $failingWorker->registeredAt, lastHeartbeat: new \DateTimeImmutable('-10 minutes'), // Stale isActive: false, // Marked as failed cpuUsage: new Percentage(99), // Critical memoryUsage: Byte::fromGigabytes(4), // Over limit currentJobs: 5 ); // Verify failure detection expect($failedWorker->isHealthy())->toBeFalse(); expect($failedWorker->isAvailableForJobs())->toBeFalse(); // Verify healthy worker is still available expect($healthyWorker->isHealthy())->toBeTrue(); expect($healthyWorker->isAvailableForJobs())->toBeTrue(); // Mock job redistribution $stmt = mock(\PDOStatement::class); $stmt->shouldReceive('execute')->andReturn(true); $stmt->shouldReceive('rowCount')->andReturn(3); // 3 jobs released $this->connection->shouldReceive('prepare')->andReturn($stmt); $releasedJobs = $this->jobDistribution->releaseAllWorkerJobs($failedWorker->id); expect($releasedJobs)->toBe(3); }); }); describe('Media Processing Pipeline Scenario', function () { it('handles resource-intensive media processing with proper load balancing', function () { // Scenario: Video streaming service processing uploads // GPU-enabled workers for video processing $videoWorkers = [ Worker::register( hostname: 'gpu-server-1', processId: 4001, queues: [QueueName::fromString('video-processing')], maxJobs: 2, // Very resource intensive capabilities: ['video-encode', 'gpu-acceleration', 'h264', 'h265'] ), Worker::register( hostname: 'gpu-server-2', processId: 4002, queues: [QueueName::fromString('video-processing')], maxJobs: 2, capabilities: ['video-encode', 'gpu-acceleration', 'h264', 'h265'] ), ]; // CPU workers for audio processing $audioWorkers = [ Worker::register( hostname: 'audio-server-1', processId: 5001, queues: [QueueName::fromString('audio-processing')], maxJobs: 8, capabilities: ['audio-encode', 'mp3', 'aac', 'flac'] ), ]; // Thumbnail generation workers $thumbnailWorkers = [ Worker::register( hostname: 'image-server-1', processId: 6001, queues: [QueueName::fromString('thumbnail-generation')], maxJobs: 10, capabilities: ['image-resize', 'ffmpeg', 'thumbnail'] ), Worker::register( hostname: 'image-server-2', processId: 6002, queues: [QueueName::fromString('thumbnail-generation')], maxJobs: 10, capabilities: ['image-resize', 'ffmpeg', 'thumbnail'] ), ]; $allWorkers = array_merge($videoWorkers, $audioWorkers, $thumbnailWorkers); // Simulate different resource usage patterns $videoWorkerUnderLoad = $videoWorkers[0]->updateHeartbeat( new Percentage(85), // High CPU for video encoding Byte::fromGigabytes(3), // High memory usage 2 // At capacity ); $audioWorkerLightLoad = $audioWorkers[0]->updateHeartbeat( new Percentage(30), // Moderate CPU Byte::fromMegabytes(800), 3 // 3/8 jobs ); $thumbnailWorkerIdle = $thumbnailWorkers[0]->updateHeartbeat( new Percentage(5), // Very low CPU Byte::fromMegabytes(200), 0 // No current jobs ); // Verify load distribution logic expect($videoWorkerUnderLoad->isAvailableForJobs())->toBeFalse(); // At capacity expect($audioWorkerLightLoad->isAvailableForJobs())->toBeTrue(); expect($thumbnailWorkerIdle->isAvailableForJobs())->toBeTrue(); // Check load percentages expect($videoWorkerUnderLoad->getLoadPercentage()->getValue())->toBe(100.0); // 2/2 jobs = 100% expect($audioWorkerLightLoad->getLoadPercentage()->getValue())->toBe(37.5); // 3/8 = 37.5% expect($thumbnailWorkerIdle->getLoadPercentage()->getValue())->toBe(5.0); // CPU load only expect(count($allWorkers))->toBe(5); }); it('prevents resource exhaustion through proper capability matching', function () { // Worker without GPU capabilities trying to handle video processing $cpuOnlyWorker = Worker::register( hostname: 'cpu-server', processId: 7001, queues: [QueueName::fromString('video-processing')], maxJobs: 10, capabilities: ['cpu-encoding'] // Missing GPU capability ); $gpuWorker = Worker::register( hostname: 'gpu-server', processId: 7002, queues: [QueueName::fromString('video-processing')], maxJobs: 2, capabilities: ['gpu-acceleration', 'video-encode', 'h264'] ); // Job requiring GPU acceleration $jobData = [ 'required_capabilities' => ['gpu-acceleration', 'h264'], 'resource_requirements' => [ 'gpu_memory' => '4GB', 'encoding_quality' => 'high', ], ]; // Mock worker scoring (would normally be done by JobDistributionService) // CPU-only worker should get score 0 (missing required capability) expect($cpuOnlyWorker->hasCapability('gpu-acceleration'))->toBeFalse(); expect($gpuWorker->hasCapability('gpu-acceleration'))->toBeTrue(); expect($gpuWorker->hasCapability('h264'))->toBeTrue(); }); }); describe('Financial Transaction Processing Scenario', function () { it('ensures transaction consistency with distributed locking', function () { // Scenario: Banking system processing concurrent transactions $transactionWorkers = [ Worker::register( hostname: 'transaction-server-1', processId: 8001, queues: [QueueName::fromString('transactions')], maxJobs: 50, // High throughput for financial data capabilities: ['payment-processing', 'fraud-detection', 'pci-compliant'] ), Worker::register( hostname: 'transaction-server-2', processId: 8002, queues: [QueueName::fromString('transactions')], maxJobs: 50, capabilities: ['payment-processing', 'fraud-detection', 'pci-compliant'] ), ]; // Simulate concurrent transaction processing $accountId = 'account-12345'; $transactionLock = LockKey::forResource('account', $accountId); // Mock lock acquisition for account processing $lockStmt = mock(\PDOStatement::class); $lockStmt->shouldReceive('execute')->andReturn(true); $lockStmt->shouldReceive('rowCount')->andReturn(1); $failLockStmt = mock(\PDOStatement::class); $failLockStmt->shouldReceive('execute')->andThrow( new \PDOException('Duplicate entry for key PRIMARY') ); $this->connection->shouldReceive('prepare')->andReturn( $lockStmt, // First worker gets lock $failLockStmt // Second worker fails ); // First worker should acquire lock successfully $worker1 = $transactionWorkers[0]; $worker2 = $transactionWorkers[1]; $firstLockResult = $this->distributedLock->acquire( $transactionLock, $worker1->id, Duration::fromMinutes(5) ); $secondLockResult = $this->distributedLock->acquire( $transactionLock, $worker2->id, Duration::fromMinutes(5) ); expect($firstLockResult)->toBeTrue(); expect($secondLockResult)->toBeFalse(); // Should fail due to existing lock }); it('handles high-frequency trading with minimal latency', function () { // High-performance workers for trading operations $tradingWorker = Worker::register( hostname: 'trading-server-hft', processId: 9001, queues: [QueueName::fromString('high-frequency-trading')], maxJobs: 1000, // Very high throughput capabilities: ['ultra-low-latency', 'market-data', 'order-execution'] ); // Simulate high load but healthy performance $performantWorker = $tradingWorker->updateHeartbeat( new Percentage(60), // Moderate CPU despite high load Byte::fromGigabytes(1.5), // Efficient memory usage 800 // High job count but within limits ); expect($performantWorker->isHealthy())->toBeTrue(); expect($performantWorker->isAvailableForJobs())->toBeTrue(); // Still has capacity expect($performantWorker->getLoadPercentage()->getValue())->toBe(80.0); // 800/1000 jobs }); }); describe('Content Delivery Network Scenario', function () { it('distributes cache warming jobs across geographic regions', function () { // Workers in different geographic regions $usEastWorkers = [ Worker::register( hostname: 'cdn-us-east-1', processId: 10001, queues: [QueueName::fromString('cache-warming')], maxJobs: 25, capabilities: ['cdn-management', 'us-east-region', 'edge-caching'] ), Worker::register( hostname: 'cdn-us-east-2', processId: 10002, queues: [QueueName::fromString('cache-warming')], maxJobs: 25, capabilities: ['cdn-management', 'us-east-region', 'edge-caching'] ), ]; $europeWorkers = [ Worker::register( hostname: 'cdn-eu-west-1', processId: 11001, queues: [QueueName::fromString('cache-warming')], maxJobs: 20, capabilities: ['cdn-management', 'eu-west-region', 'edge-caching'] ), ]; $asiaWorkers = [ Worker::register( hostname: 'cdn-asia-pacific-1', processId: 12001, queues: [QueueName::fromString('cache-warming')], maxJobs: 15, capabilities: ['cdn-management', 'asia-pacific-region', 'edge-caching'] ), ]; $allCdnWorkers = array_merge($usEastWorkers, $europeWorkers, $asiaWorkers); // Verify regional distribution $usEastCount = count(array_filter( $allCdnWorkers, fn ($w) => $w->hasCapability('us-east-region') )); $europeCount = count(array_filter( $allCdnWorkers, fn ($w) => $w->hasCapability('eu-west-region') )); $asiaCount = count(array_filter( $allCdnWorkers, fn ($w) => $w->hasCapability('asia-pacific-region') )); expect($usEastCount)->toBe(2); expect($europeCount)->toBe(1); expect($asiaCount)->toBe(1); // Verify all workers can handle cache warming foreach ($allCdnWorkers as $worker) { expect($worker->hasCapability('cdn-management'))->toBeTrue(); expect($worker->hasCapability('edge-caching'))->toBeTrue(); } }); it('handles regional worker failure with graceful degradation', function () { // Scenario: Entire region goes offline, traffic redistributed $primaryWorker = Worker::register( hostname: 'cdn-primary', processId: 13001, queues: [QueueName::fromString('content-delivery')], maxJobs: 100, capabilities: ['primary-region', 'high-capacity'] ); $backupWorkers = [ Worker::register( hostname: 'cdn-backup-1', processId: 13002, queues: [QueueName::fromString('content-delivery')], maxJobs: 50, capabilities: ['backup-region', 'medium-capacity'] ), Worker::register( hostname: 'cdn-backup-2', processId: 13003, queues: [QueueName::fromString('content-delivery')], maxJobs: 50, capabilities: ['backup-region', 'medium-capacity'] ), ]; // Simulate primary region failure $failedPrimary = $primaryWorker->markInactive(); expect($failedPrimary->isAvailableForJobs())->toBeFalse(); // Backup workers should still be available foreach ($backupWorkers as $backup) { expect($backup->isAvailableForJobs())->toBeTrue(); } // Total backup capacity should handle reduced load $totalBackupCapacity = array_sum(array_map(fn ($w) => $w->maxJobs, $backupWorkers)); expect($totalBackupCapacity)->toBe(100); // Same as primary capacity }); }); describe('Machine Learning Training Pipeline Scenario', function () { it('manages resource-intensive ML training jobs efficiently', function () { // Specialized workers for different ML tasks $gpuTrainingWorker = Worker::register( hostname: 'ml-gpu-cluster-1', processId: 14001, queues: [QueueName::fromString('ml-training')], maxJobs: 1, // One intensive job at a time capabilities: ['gpu-cluster', 'tensorflow', 'pytorch', 'cuda'] ); $dataPreprocessingWorkers = [ Worker::register( hostname: 'ml-preprocessing-1', processId: 14002, queues: [QueueName::fromString('data-preprocessing')], maxJobs: 10, capabilities: ['data-cleaning', 'feature-engineering', 'pandas', 'numpy'] ), Worker::register( hostname: 'ml-preprocessing-2', processId: 14003, queues: [QueueName::fromString('data-preprocessing')], maxJobs: 10, capabilities: ['data-cleaning', 'feature-engineering', 'pandas', 'numpy'] ), ]; $inferenceWorkers = [ Worker::register( hostname: 'ml-inference-1', processId: 14004, queues: [QueueName::fromString('ml-inference')], maxJobs: 50, // High throughput for inference capabilities: ['model-serving', 'tensorflow-lite', 'onnx'] ), ]; // Simulate GPU worker under heavy load $trainingWorkerLoaded = $gpuTrainingWorker->updateHeartbeat( new Percentage(95), // High GPU utilization Byte::fromGigabytes(15), // High memory for large models 1 // At capacity ); // Preprocessing workers with moderate load $preprocessingWorkerActive = $dataPreprocessingWorkers[0]->updateHeartbeat( new Percentage(70), // Active data processing Byte::fromGigabytes(4), 6 // 6/10 jobs ); // Inference worker with light load $inferenceWorkerIdle = $inferenceWorkers[0]->updateHeartbeat( new Percentage(20), // Low CPU for inference Byte::fromMegabytes(800), 5 // 5/50 jobs ); // Verify resource allocation patterns expect($trainingWorkerLoaded->isAvailableForJobs())->toBeFalse(); // At capacity expect($preprocessingWorkerActive->isAvailableForJobs())->toBeTrue(); expect($inferenceWorkerIdle->isAvailableForJobs())->toBeTrue(); // Check load patterns match expected ML workloads expect($trainingWorkerLoaded->getLoadPercentage()->getValue())->toBe(100.0); // At capacity expect($preprocessingWorkerActive->getLoadPercentage()->getValue())->toBe(70.0); // CPU bound expect($inferenceWorkerIdle->getLoadPercentage()->getValue())->toBe(20.0); // Light load }); }); });