database = $this->createTestDatabase(); $this->workerRegistry = new WorkerRegistry($this->database); $this->distributionService = new JobDistributionService( $this->database, $this->workerRegistry ); $this->healthCheckService = new WorkerHealthCheckService( $this->database, $this->workerRegistry ); $this->failoverService = new FailoverRecoveryService( $this->database, $this->workerRegistry ); $this->cleanupTestData(); PerformanceTestHelper::warmupDatabase($this->database->getConnection()); } protected function tearDown(): void { $this->cleanupTestData(); } public function testEcommercePeakTrafficScenario(): void { echo "\nE-commerce Peak Traffic Scenario:\n"; echo "Simulating Black Friday / Cyber Monday traffic patterns\n"; // Setup: Mixed capacity workers for different job types $workers = [ // High-capacity workers for order processing ...$this->createWorkers(8, 50, 'order_processor'), // Medium-capacity workers for inventory updates ...$this->createWorkers(12, 30, 'inventory_worker'), // Lower-capacity workers for email notifications ...$this->createWorkers(20, 15, 'notification_worker'), ]; $this->registerWorkers($workers); // Simulate peak traffic: 1000+ jobs per minute for 5 minutes $scenarioDuration = 300; // 5 minutes $peakJobsPerMinute = 1200; $jobsPerSecond = $peakJobsPerMinute / 60; echo "Target load: {$peakJobsPerMinute} jobs/minute ({$jobsPerSecond} jobs/second)\n"; echo "Duration: {$scenarioDuration} seconds\n"; $results = $this->simulateRealisticLoad( $scenarioDuration, $jobsPerSecond, $this->createEcommerceJobMix() ); echo "\nE-commerce Peak Results:\n"; echo "Actual throughput: {$results['throughput']} jobs/second\n"; echo "Success rate: {$results['success_rate']}%\n"; echo "Average response time: {$results['avg_response_time']}ms\n"; echo "P95 response time: {$results['p95_response_time']}ms\n"; echo "Memory usage: {$results['memory_usage']}MB\n"; // Validate e-commerce performance requirements $this->assertGreaterThan(15, $results['throughput'], 'E-commerce throughput below minimum'); $this->assertGreaterThan(95.0, $results['success_rate'], 'E-commerce success rate below 95%'); $this->assertLessThan(100.0, $results['avg_response_time'], 'E-commerce response time too high'); $this->assertLessThan(200.0, $results['p95_response_time'], 'E-commerce P95 response time too high'); } public function testMediaProcessingWorkloadScenario(): void { echo "\nMedia Processing Workload Scenario:\n"; echo "Simulating video transcoding and image processing pipeline\n"; // Setup: Fewer, high-capacity workers for CPU/memory intensive tasks $workers = [ // Heavy-duty workers for video processing ...$this->createWorkers(4, 100, 'video_processor'), // Medium workers for image processing ...$this->createWorkers(8, 50, 'image_processor'), // Light workers for metadata extraction ...$this->createWorkers(12, 25, 'metadata_worker'), ]; $this->registerWorkers($workers); // Simulate media processing: Lower frequency but CPU/memory intensive $scenarioDuration = 600; // 10 minutes $jobsPerSecond = 3; // Lower rate due to intensive processing echo "Target load: {$jobsPerSecond} jobs/second (CPU/memory intensive)\n"; echo "Duration: {$scenarioDuration} seconds\n"; $results = $this->simulateRealisticLoad( $scenarioDuration, $jobsPerSecond, $this->createMediaProcessingJobMix(), $enableResourceMonitoring = true ); echo "\nMedia Processing Results:\n"; echo "Actual throughput: {$results['throughput']} jobs/second\n"; echo "Success rate: {$results['success_rate']}%\n"; echo "Average processing time: {$results['avg_response_time']}ms\n"; echo "Memory efficiency: {$results['memory_efficiency']}%\n"; echo "Resource utilization: {$results['resource_utilization']}%\n"; // Validate media processing performance requirements $this->assertGreaterThan(2.5, $results['throughput'], 'Media processing throughput below minimum'); $this->assertGreaterThan(98.0, $results['success_rate'], 'Media processing success rate below 98%'); $this->assertLessThan(500.0, $results['avg_response_time'], 'Media processing time too high'); $this->assertGreaterThan(80.0, $results['resource_utilization'], 'Resource utilization too low'); } public function testFinancialTransactionProcessingScenario(): void { echo "\nFinancial Transaction Processing Scenario:\n"; echo "Simulating real-time payment processing with low latency requirements\n"; // Setup: Many workers optimized for low-latency processing $workers = [ // High-speed transaction processors ...$this->createWorkers(20, 20, 'payment_processor'), // Fraud detection workers ...$this->createWorkers(10, 15, 'fraud_detector'), // Settlement workers ...$this->createWorkers(5, 30, 'settlement_worker'), ]; $this->registerWorkers($workers); // Simulate financial processing: High frequency, low latency requirements $scenarioDuration = 120; // 2 minutes $jobsPerSecond = 50; // High frequency transactions echo "Target load: {$jobsPerSecond} jobs/second (low latency requirement)\n"; echo "Duration: {$scenarioDuration} seconds\n"; $results = $this->simulateRealisticLoad( $scenarioDuration, $jobsPerSecond, $this->createFinancialJobMix(), $enableResourceMonitoring = false, $lowLatencyMode = true ); echo "\nFinancial Processing Results:\n"; echo "Actual throughput: {$results['throughput']} jobs/second\n"; echo "Success rate: {$results['success_rate']}%\n"; echo "Average latency: {$results['avg_response_time']}ms\n"; echo "P95 latency: {$results['p95_response_time']}ms\n"; echo "P99 latency: {$results['p99_response_time']}ms\n"; // Validate financial processing requirements (strict latency) $this->assertGreaterThan(40, $results['throughput'], 'Financial throughput below minimum'); $this->assertGreaterThan(99.9, $results['success_rate'], 'Financial success rate below 99.9%'); $this->assertLessThan(20.0, $results['avg_response_time'], 'Financial latency too high'); $this->assertLessThan(50.0, $results['p95_response_time'], 'Financial P95 latency too high'); $this->assertLessThan(100.0, $results['p99_response_time'], 'Financial P99 latency too high'); } public function testBatchProcessingScenario(): void { echo "\nBatch Processing Scenario:\n"; echo "Simulating ETL pipeline with high throughput requirements\n"; // Setup: High-capacity workers optimized for batch processing $workers = [ // ETL workers for data transformation ...$this->createWorkers(6, 100, 'etl_worker'), // Data validation workers ...$this->createWorkers(8, 75, 'validator'), // Report generation workers ...$this->createWorkers(4, 150, 'report_generator'), ]; $this->registerWorkers($workers); // Simulate batch processing: Very high throughput $scenarioDuration = 300; // 5 minutes $jobsPerSecond = 100; // High throughput batch processing echo "Target load: {$jobsPerSecond} jobs/second (high throughput batch)\n"; echo "Duration: {$scenarioDuration} seconds\n"; $results = $this->simulateRealisticLoad( $scenarioDuration, $jobsPerSecond, $this->createBatchProcessingJobMix(), $enableResourceMonitoring = true ); echo "\nBatch Processing Results:\n"; echo "Actual throughput: {$results['throughput']} jobs/second\n"; echo "Success rate: {$results['success_rate']}%\n"; echo "Batch efficiency: {$results['batch_efficiency']}%\n"; echo "Resource utilization: {$results['resource_utilization']}%\n"; echo "Memory stability: {$results['memory_stability']}\n"; // Validate batch processing requirements $this->assertGreaterThan(80, $results['throughput'], 'Batch throughput below minimum'); $this->assertGreaterThan(99.0, $results['success_rate'], 'Batch success rate below 99%'); $this->assertGreaterThan(85.0, $results['batch_efficiency'], 'Batch efficiency too low'); $this->assertGreaterThan(75.0, $results['resource_utilization'], 'Batch resource utilization too low'); } public function testMixedWorkloadStressTest(): void { echo "\nMixed Workload Stress Test:\n"; echo "Simulating real-world environment with multiple concurrent workload types\n"; // Setup: Diverse worker pool handling multiple workload types $workers = [ // Web request processors ...$this->createWorkers(15, 30, 'web_processor'), // Background task workers ...$this->createWorkers(10, 20, 'background_worker'), // Heavy computation workers ...$this->createWorkers(5, 80, 'compute_worker'), // Notification workers ...$this->createWorkers(20, 10, 'notification_worker'), ]; $this->registerWorkers($workers); // Simulate mixed workload with varying intensity $phases = [ ['duration' => 60, 'rate' => 20, 'mix' => 'normal'], ['duration' => 120, 'rate' => 50, 'mix' => 'peak'], ['duration' => 60, 'rate' => 15, 'mix' => 'background'], ['duration' => 90, 'rate' => 35, 'mix' => 'mixed'], ]; $overallResults = []; foreach ($phases as $phaseIndex => $phase) { echo "\nPhase " . ($phaseIndex + 1) . ": {$phase['mix']} workload\n"; echo "Duration: {$phase['duration']}s, Rate: {$phase['rate']} jobs/sec\n"; $jobMix = $this->createMixedWorkloadJobMix($phase['mix']); $results = $this->simulateRealisticLoad( $phase['duration'], $phase['rate'], $jobMix, $enableResourceMonitoring = true ); echo "Phase Results - Throughput: {$results['throughput']}, Success: {$results['success_rate']}%\n"; $overallResults[] = $results; // Brief pause between phases sleep(2); } // Analyze overall performance across all phases $this->analyzeOverallPerformance($overallResults); } public function testFailoverUnderRealWorldLoad(): void { echo "\nFailover Under Real-World Load Test:\n"; echo "Simulating worker failures during active production load\n"; // Setup: Production-like worker configuration $workers = [ ...$this->createWorkers(12, 25, 'primary_worker'), ...$this->createWorkers(8, 30, 'secondary_worker'), ...$this->createWorkers(6, 20, 'backup_worker'), ]; $this->registerWorkers($workers); // Start sustained load $testDuration = 180; // 3 minutes $baseJobRate = 30; // jobs per second echo "Base load: {$baseJobRate} jobs/second\n"; echo "Test duration: {$testDuration} seconds\n"; $startTime = microtime(true); $endTime = $startTime + $testDuration; $metrics = [ 'jobs_processed' => 0, 'jobs_failed' => 0, 'response_times' => [], 'failover_events' => [], ]; $failoverTriggered = false; while (microtime(true) < $endTime) { $cycleStart = microtime(true); // Trigger failover at 1/3 of test duration if (! $failoverTriggered && (microtime(true) - $startTime) > ($testDuration / 3)) { echo "\nTriggering failover scenario...\n"; // Fail primary workers for ($i = 1; $i <= 4; $i++) { $this->updateWorkerStatus("primary_worker_{$i}", WorkerStatus::FAILED); } $failoverTime = PerformanceTestHelper::measureTime(function () { $this->failoverService->performFullSystemRecovery(); }); $metrics['failover_events'][] = [ 'time' => microtime(true) - $startTime, 'recovery_time' => $failoverTime, ]; echo "Failover completed in {$failoverTime}ms\n"; $failoverTriggered = true; } // Process jobs for ($i = 0; $i < $baseJobRate; $i++) { $job = PerformanceTestHelper::createTestJob("realworld_job_{$metrics['jobs_processed']}"); $result = PerformanceTestHelper::measureTimeWithResult(function () use ($job) { try { return $this->distributionService->distributeJob($job); } catch (\Exception $e) { return null; } }); $metrics['response_times'][] = $result['time_ms']; if ($result['result'] !== null) { $metrics['jobs_processed']++; } else { $metrics['jobs_failed']++; } } // Maintain rate $cycleTime = microtime(true) - $cycleStart; $sleepTime = 1.0 - $cycleTime; if ($sleepTime > 0) { usleep($sleepTime * 1000000); } } $actualDuration = microtime(true) - $startTime; $actualThroughput = $metrics['jobs_processed'] / $actualDuration; $successRate = $metrics['jobs_processed'] / ($metrics['jobs_processed'] + $metrics['jobs_failed']) * 100; $responseStats = PerformanceTestHelper::calculateStatistics($metrics['response_times']); echo "\nFailover Test Results:\n"; echo "Actual throughput: {$actualThroughput} jobs/second\n"; echo "Success rate: {$successRate}%\n"; echo "Response times: " . PerformanceTestHelper::formatStatistics($responseStats) . "\n"; if (! empty($metrics['failover_events'])) { echo "Failover recovery time: {$metrics['failover_events'][0]['recovery_time']}ms\n"; } // System should maintain reasonable performance during failover $this->assertGreaterThan(20, $actualThroughput, 'Throughput too low during failover'); $this->assertGreaterThan(90.0, $successRate, 'Success rate too low during failover'); $this->assertLessThan(100.0, $responseStats['avg'], 'Response time too high during failover'); } private function simulateRealisticLoad( int $duration, float $jobsPerSecond, array $jobMix, bool $enableResourceMonitoring = false, bool $lowLatencyMode = false ): array { $startTime = microtime(true); $endTime = $startTime + $duration; $metrics = [ 'jobs_processed' => 0, 'jobs_failed' => 0, 'response_times' => [], 'memory_snapshots' => [], 'start_memory' => null, 'end_memory' => null, ]; if ($enableResourceMonitoring) { $metrics['start_memory'] = PerformanceTestHelper::getMemoryUsage(); } $jobCounter = 0; $snapshotInterval = $enableResourceMonitoring ? 30 : 0; // Take snapshots every 30 seconds $nextSnapshotTime = $startTime + $snapshotInterval; while (microtime(true) < $endTime) { $cycleStart = microtime(true); // Determine job type based on mix $jobType = $this->selectJobType($jobMix); $job = $this->createJobForType($jobType, $jobCounter); $result = PerformanceTestHelper::measureTimeWithResult(function () use ($job) { try { return $this->distributionService->distributeJob($job); } catch (\Exception $e) { return null; } }); $metrics['response_times'][] = $result['time_ms']; if ($result['result'] !== null) { $metrics['jobs_processed']++; } else { $metrics['jobs_failed']++; } $jobCounter++; // Take memory snapshots if ($enableResourceMonitoring && microtime(true) >= $nextSnapshotTime) { $metrics['memory_snapshots'][] = [ 'time' => microtime(true) - $startTime, 'memory' => PerformanceTestHelper::getMemoryUsage(), ]; $nextSnapshotTime += $snapshotInterval; } // Rate limiting if ($lowLatencyMode) { // Minimal delay for low latency requirements usleep(10); // 0.01ms } else { // Calculate delay to maintain target rate $targetCycleTime = 1.0 / $jobsPerSecond; $actualCycleTime = microtime(true) - $cycleStart; $sleepTime = $targetCycleTime - $actualCycleTime; if ($sleepTime > 0) { usleep($sleepTime * 1000000); } } } if ($enableResourceMonitoring) { $metrics['end_memory'] = PerformanceTestHelper::getMemoryUsage(); } return $this->calculateScenarioResults($metrics, microtime(true) - $startTime, $enableResourceMonitoring); } private function calculateScenarioResults(array $metrics, float $actualDuration, bool $includeResourceMetrics): array { $throughput = $metrics['jobs_processed'] / $actualDuration; $successRate = $metrics['jobs_processed'] / max(1, $metrics['jobs_processed'] + $metrics['jobs_failed']) * 100; $responseStats = PerformanceTestHelper::calculateStatistics($metrics['response_times']); $results = [ 'throughput' => round($throughput, 1), 'success_rate' => round($successRate, 2), 'avg_response_time' => $responseStats['avg'], 'p95_response_time' => $responseStats['p95'], 'p99_response_time' => $responseStats['p99'], ]; if ($includeResourceMetrics && isset($metrics['start_memory'], $metrics['end_memory'])) { $startMem = $metrics['start_memory']['current_mb']; $endMem = $metrics['end_memory']['current_mb']; $peakMem = $metrics['end_memory']['peak_mb']; $results['memory_usage'] = $endMem; $results['memory_efficiency'] = round((1 - ($endMem - $startMem) / max(1, $startMem)) * 100, 1); $results['resource_utilization'] = round(($endMem / $peakMem) * 100, 1); $results['memory_stability'] = abs($endMem - $startMem) < 10 ? 'stable' : 'unstable'; $results['batch_efficiency'] = round($throughput / max(1, $endMem) * 100, 1); } return $results; } private function createEcommerceJobMix(): array { return [ 'order_processing' => 40, 'inventory_update' => 25, 'payment_processing' => 20, 'email_notification' => 10, 'user_analytics' => 5, ]; } private function createMediaProcessingJobMix(): array { return [ 'video_transcode' => 30, 'image_resize' => 40, 'thumbnail_generation' => 20, 'metadata_extraction' => 10, ]; } private function createFinancialJobMix(): array { return [ 'payment_processing' => 50, 'fraud_detection' => 25, 'account_verification' => 15, 'transaction_logging' => 10, ]; } private function createBatchProcessingJobMix(): array { return [ 'data_transformation' => 40, 'data_validation' => 30, 'report_generation' => 20, 'data_archival' => 10, ]; } private function createMixedWorkloadJobMix(string $mixType): array { return match($mixType) { 'normal' => [ 'web_request' => 50, 'background_task' => 30, 'notification' => 20, ], 'peak' => [ 'web_request' => 60, 'background_task' => 20, 'notification' => 15, 'compute_task' => 5, ], 'background' => [ 'background_task' => 60, 'compute_task' => 30, 'notification' => 10, ], 'mixed' => [ 'web_request' => 35, 'background_task' => 25, 'compute_task' => 25, 'notification' => 15, ], default => ['web_request' => 100] }; } private function selectJobType(array $jobMix): string { $rand = rand(1, 100); $cumulative = 0; foreach ($jobMix as $type => $percentage) { $cumulative += $percentage; if ($rand <= $cumulative) { return $type; } } return array_key_first($jobMix); } private function createJobForType(string $jobType, int $counter): \App\Framework\Queue\Jobs\Job { $priority = match($jobType) { 'payment_processing', 'fraud_detection' => JobPriority::CRITICAL, 'order_processing', 'web_request' => JobPriority::HIGH, 'inventory_update', 'background_task' => JobPriority::NORMAL, default => JobPriority::LOW }; $payloadSize = match($jobType) { 'video_transcode', 'compute_task' => 1000, // Large payload 'image_resize', 'data_transformation' => 500, // Medium payload default => 100 // Small payload }; return PerformanceTestHelper::createTestJob( "{$jobType}_job_{$counter}", $priority, ['type' => $jobType, 'data' => str_repeat('x', $payloadSize)] ); } private function analyzeOverallPerformance(array $phaseResults): void { echo "\nOverall Mixed Workload Analysis:\n"; $totalThroughput = array_sum(array_column($phaseResults, 'throughput')) / count($phaseResults); $averageSuccessRate = array_sum(array_column($phaseResults, 'success_rate')) / count($phaseResults); $averageResponseTime = array_sum(array_column($phaseResults, 'avg_response_time')) / count($phaseResults); echo "Average throughput across phases: {$totalThroughput} jobs/second\n"; echo "Average success rate: {$averageSuccessRate}%\n"; echo "Average response time: {$averageResponseTime}ms\n"; // Validate mixed workload performance $this->assertGreaterThan(25, $totalThroughput, 'Mixed workload throughput below minimum'); $this->assertGreaterThan(95.0, $averageSuccessRate, 'Mixed workload success rate below 95%'); $this->assertLessThan(80.0, $averageResponseTime, 'Mixed workload response time too high'); // Check performance consistency across phases $throughputStdDev = $this->calculateStandardDeviation(array_column($phaseResults, 'throughput')); $this->assertLessThan(10.0, $throughputStdDev, 'Throughput too inconsistent across phases'); } private function calculateStandardDeviation(array $values): float { $mean = array_sum($values) / count($values); $sumSquaredDiffs = array_sum(array_map(fn ($v) => pow($v - $mean, 2), $values)); return sqrt($sumSquaredDiffs / count($values)); } private function createWorkers(int $count, int $capacity, string $prefix): array { $workers = []; for ($i = 1; $i <= $count; $i++) { $workers[] = PerformanceTestHelper::createTestWorker( "{$prefix}_{$i}", $capacity, WorkerStatus::AVAILABLE ); } return $workers; } private function registerWorkers(array $workers): void { foreach ($workers as $worker) { $this->workerRegistry->registerWorker($worker); } } private function updateWorkerStatus(string $workerId, WorkerStatus $status): void { $pdo = $this->database->getConnection(); $stmt = $pdo->prepare('UPDATE workers SET status = ? WHERE id = ?'); $stmt->execute([$status->value, $workerId]); } private function createTestDatabase(): DatabaseManager { $pdo = new \PDO('sqlite::memory:'); $pdo->setAttribute(\PDO::ATTR_ERRMODE, \PDO::ERRMODE_EXCEPTION); $pdo->exec(' CREATE TABLE workers ( id TEXT PRIMARY KEY, queue_names TEXT NOT NULL, capacity INTEGER NOT NULL, status TEXT NOT NULL, last_heartbeat TEXT NOT NULL, metadata TEXT ) '); $pdo->exec(' CREATE TABLE jobs ( id TEXT PRIMARY KEY, type TEXT NOT NULL, payload TEXT NOT NULL, queue_name TEXT NOT NULL, priority INTEGER NOT NULL, status TEXT NOT NULL, worker_id TEXT, created_at TEXT NOT NULL, started_at TEXT, completed_at TEXT, attempts INTEGER DEFAULT 0, error_message TEXT ) '); // Performance indexes $pdo->exec('CREATE INDEX idx_workers_status ON workers(status)'); $pdo->exec('CREATE INDEX idx_jobs_status ON jobs(status)'); $pdo->exec('CREATE INDEX idx_jobs_priority ON jobs(priority)'); $pdo->exec('CREATE INDEX idx_jobs_worker ON jobs(worker_id)'); return new DatabaseManager($pdo); } private function cleanupTestData(): void { $pdo = $this->database->getConnection(); $pdo->exec('DELETE FROM workers'); $pdo->exec('DELETE FROM jobs'); } }