Michael Schiemer
fc3d7e6357
- Add comprehensive health check system with multiple endpoints - Add Prometheus metrics endpoint - Add production logging configurations (5 strategies) - Add complete deployment documentation suite: * QUICKSTART.md - 30-minute deployment guide * DEPLOYMENT_CHECKLIST.md - Printable verification checklist * DEPLOYMENT_WORKFLOW.md - Complete deployment lifecycle * PRODUCTION_DEPLOYMENT.md - Comprehensive technical reference * production-logging.md - Logging configuration guide * ANSIBLE_DEPLOYMENT.md - Infrastructure as Code automation * README.md - Navigation hub * DEPLOYMENT_SUMMARY.md - Executive summary - Add deployment scripts and automation - Add DEPLOYMENT_PLAN.md - Concrete plan for immediate deployment - Update README with production-ready features All production infrastructure is now complete and ready for deployment.
587 lines
25 KiB
PHP
587 lines
25 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use App\Framework\Core\ValueObjects\Byte;
|
|
use App\Framework\Core\ValueObjects\Duration;
|
|
use App\Framework\Core\ValueObjects\Percentage;
|
|
use App\Framework\Database\ConnectionInterface;
|
|
use App\Framework\Logging\Logger;
|
|
use App\Framework\Queue\Entities\Worker;
|
|
use App\Framework\Queue\Services\DatabaseDistributedLock;
|
|
use App\Framework\Queue\Services\JobDistributionService;
|
|
use App\Framework\Queue\Services\WorkerRegistry;
|
|
use App\Framework\Queue\ValueObjects\JobId;
|
|
use App\Framework\Queue\ValueObjects\LockKey;
|
|
use App\Framework\Queue\ValueObjects\QueueName;
|
|
|
|
/**
|
|
* Real-world scenario tests for the Distributed Processing System
|
|
* These tests simulate complex real-world scenarios to validate system behavior
|
|
*/
|
|
describe('Distributed Processing Real-World Scenarios', function () {
|
|
beforeEach(function () {
|
|
// Mock connection and logger
|
|
$this->connection = mock(ConnectionInterface::class);
|
|
$this->logger = mock(Logger::class);
|
|
|
|
// Setup services
|
|
$this->workerRegistry = new WorkerRegistry($this->connection, $this->logger);
|
|
$this->distributedLock = new DatabaseDistributedLock($this->connection, $this->logger);
|
|
$this->jobDistribution = new JobDistributionService(
|
|
$this->workerRegistry,
|
|
$this->distributedLock,
|
|
$this->connection,
|
|
$this->logger
|
|
);
|
|
|
|
// Mock default logger behavior
|
|
$this->logger->shouldReceive('info')->andReturn(null);
|
|
$this->logger->shouldReceive('debug')->andReturn(null);
|
|
$this->logger->shouldReceive('warning')->andReturn(null);
|
|
$this->logger->shouldReceive('error')->andReturn(null);
|
|
});
|
|
|
|
describe('E-commerce Order Processing Scenario', function () {
|
|
it('handles peak shopping season with multiple worker types', function () {
|
|
// Scenario: Black Friday traffic with specialized workers
|
|
|
|
// Create specialized workers for different tasks
|
|
$emailWorkers = [
|
|
Worker::register(
|
|
hostname: 'email-server-1',
|
|
processId: 2001,
|
|
queues: [QueueName::emailQueue()],
|
|
maxJobs: 20,
|
|
capabilities: ['email', 'newsletter', 'notifications']
|
|
),
|
|
Worker::register(
|
|
hostname: 'email-server-2',
|
|
processId: 2002,
|
|
queues: [QueueName::emailQueue()],
|
|
maxJobs: 20,
|
|
capabilities: ['email', 'newsletter', 'notifications']
|
|
),
|
|
];
|
|
|
|
$imageWorkers = [
|
|
Worker::register(
|
|
hostname: 'image-server-1',
|
|
processId: 3001,
|
|
queues: [QueueName::fromString('image-processing')],
|
|
maxJobs: 5, // Resource intensive
|
|
capabilities: ['image-resize', 'thumbnail', 'watermark']
|
|
),
|
|
];
|
|
|
|
$generalWorkers = [
|
|
Worker::register(
|
|
hostname: 'app-server-1',
|
|
processId: 1001,
|
|
queues: [
|
|
QueueName::defaultQueue(),
|
|
QueueName::fromString('reports'),
|
|
],
|
|
maxJobs: 15,
|
|
capabilities: ['pdf-generation', 'reporting', 'exports']
|
|
),
|
|
Worker::register(
|
|
hostname: 'app-server-2',
|
|
processId: 1002,
|
|
queues: [
|
|
QueueName::defaultQueue(),
|
|
QueueName::fromString('reports'),
|
|
],
|
|
maxJobs: 15,
|
|
capabilities: ['pdf-generation', 'reporting', 'exports']
|
|
),
|
|
];
|
|
|
|
$allWorkers = array_merge($emailWorkers, $imageWorkers, $generalWorkers);
|
|
|
|
// Mock worker registration
|
|
foreach ($allWorkers as $worker) {
|
|
$stmt = mock(\PDOStatement::class);
|
|
$stmt->shouldReceive('execute')->andReturn(true);
|
|
$this->connection->shouldReceive('prepare')->andReturn($stmt);
|
|
|
|
$this->workerRegistry->register($worker);
|
|
}
|
|
|
|
// Simulate different job types being distributed
|
|
$jobs = [
|
|
// Email jobs (high volume, low resource)
|
|
['id' => JobId::generate(), 'queue' => QueueName::emailQueue(), 'type' => 'order-confirmation'],
|
|
['id' => JobId::generate(), 'queue' => QueueName::emailQueue(), 'type' => 'shipping-notification'],
|
|
['id' => JobId::generate(), 'queue' => QueueName::emailQueue(), 'type' => 'newsletter'],
|
|
|
|
// Image processing jobs (low volume, high resource)
|
|
['id' => JobId::generate(), 'queue' => QueueName::fromString('image-processing'), 'type' => 'product-thumbnails'],
|
|
|
|
// General processing jobs
|
|
['id' => JobId::generate(), 'queue' => QueueName::defaultQueue(), 'type' => 'invoice-generation'],
|
|
['id' => JobId::generate(), 'queue' => QueueName::fromString('reports'), 'type' => 'sales-report'],
|
|
];
|
|
|
|
// Mock job distribution
|
|
foreach ($jobs as $job) {
|
|
// Mock finding workers for queue
|
|
$workerStmt = mock(\PDOStatement::class);
|
|
$workerStmt->shouldReceive('execute')->andReturn(true);
|
|
$workerStmt->shouldReceive('fetch')->andReturn(false); // Simplified for test
|
|
|
|
// Mock lock operations
|
|
$lockStmt = mock(\PDOStatement::class);
|
|
$lockStmt->shouldReceive('execute')->andReturn(true);
|
|
$lockStmt->shouldReceive('rowCount')->andReturn(1);
|
|
|
|
$this->connection->shouldReceive('prepare')->andReturn($workerStmt, $lockStmt);
|
|
|
|
$assignedWorker = $this->jobDistribution->findBestWorkerForJob($job['queue']);
|
|
// In real scenario, would validate worker assignment logic
|
|
}
|
|
|
|
// Verify system can handle the load
|
|
expect(count($allWorkers))->toBe(5);
|
|
expect(count($jobs))->toBe(6);
|
|
});
|
|
|
|
it('handles worker failure during peak traffic gracefully', function () {
|
|
// Scenario: Worker crashes during high load, jobs need redistribution
|
|
|
|
$healthyWorker = Worker::register(
|
|
hostname: 'stable-server',
|
|
processId: 1001,
|
|
queues: [QueueName::defaultQueue()],
|
|
maxJobs: 10,
|
|
capabilities: ['email', 'pdf-generation']
|
|
);
|
|
|
|
$failingWorker = Worker::register(
|
|
hostname: 'failing-server',
|
|
processId: 1002,
|
|
queues: [QueueName::defaultQueue()],
|
|
maxJobs: 10,
|
|
capabilities: ['email', 'pdf-generation']
|
|
);
|
|
|
|
// Simulate worker failure (stale heartbeat, high resource usage)
|
|
$failedWorker = new Worker(
|
|
id: $failingWorker->id,
|
|
hostname: $failingWorker->hostname,
|
|
processId: $failingWorker->processId,
|
|
queues: $failingWorker->queues,
|
|
maxJobs: $failingWorker->maxJobs,
|
|
registeredAt: $failingWorker->registeredAt,
|
|
lastHeartbeat: new \DateTimeImmutable('-10 minutes'), // Stale
|
|
isActive: false, // Marked as failed
|
|
cpuUsage: new Percentage(99), // Critical
|
|
memoryUsage: Byte::fromGigabytes(4), // Over limit
|
|
currentJobs: 5
|
|
);
|
|
|
|
// Verify failure detection
|
|
expect($failedWorker->isHealthy())->toBeFalse();
|
|
expect($failedWorker->isAvailableForJobs())->toBeFalse();
|
|
|
|
// Verify healthy worker is still available
|
|
expect($healthyWorker->isHealthy())->toBeTrue();
|
|
expect($healthyWorker->isAvailableForJobs())->toBeTrue();
|
|
|
|
// Mock job redistribution
|
|
$stmt = mock(\PDOStatement::class);
|
|
$stmt->shouldReceive('execute')->andReturn(true);
|
|
$stmt->shouldReceive('rowCount')->andReturn(3); // 3 jobs released
|
|
|
|
$this->connection->shouldReceive('prepare')->andReturn($stmt);
|
|
|
|
$releasedJobs = $this->jobDistribution->releaseAllWorkerJobs($failedWorker->id);
|
|
expect($releasedJobs)->toBe(3);
|
|
});
|
|
});
|
|
|
|
describe('Media Processing Pipeline Scenario', function () {
|
|
it('handles resource-intensive media processing with proper load balancing', function () {
|
|
// Scenario: Video streaming service processing uploads
|
|
|
|
// GPU-enabled workers for video processing
|
|
$videoWorkers = [
|
|
Worker::register(
|
|
hostname: 'gpu-server-1',
|
|
processId: 4001,
|
|
queues: [QueueName::fromString('video-processing')],
|
|
maxJobs: 2, // Very resource intensive
|
|
capabilities: ['video-encode', 'gpu-acceleration', 'h264', 'h265']
|
|
),
|
|
Worker::register(
|
|
hostname: 'gpu-server-2',
|
|
processId: 4002,
|
|
queues: [QueueName::fromString('video-processing')],
|
|
maxJobs: 2,
|
|
capabilities: ['video-encode', 'gpu-acceleration', 'h264', 'h265']
|
|
),
|
|
];
|
|
|
|
// CPU workers for audio processing
|
|
$audioWorkers = [
|
|
Worker::register(
|
|
hostname: 'audio-server-1',
|
|
processId: 5001,
|
|
queues: [QueueName::fromString('audio-processing')],
|
|
maxJobs: 8,
|
|
capabilities: ['audio-encode', 'mp3', 'aac', 'flac']
|
|
),
|
|
];
|
|
|
|
// Thumbnail generation workers
|
|
$thumbnailWorkers = [
|
|
Worker::register(
|
|
hostname: 'image-server-1',
|
|
processId: 6001,
|
|
queues: [QueueName::fromString('thumbnail-generation')],
|
|
maxJobs: 10,
|
|
capabilities: ['image-resize', 'ffmpeg', 'thumbnail']
|
|
),
|
|
Worker::register(
|
|
hostname: 'image-server-2',
|
|
processId: 6002,
|
|
queues: [QueueName::fromString('thumbnail-generation')],
|
|
maxJobs: 10,
|
|
capabilities: ['image-resize', 'ffmpeg', 'thumbnail']
|
|
),
|
|
];
|
|
|
|
$allWorkers = array_merge($videoWorkers, $audioWorkers, $thumbnailWorkers);
|
|
|
|
// Simulate different resource usage patterns
|
|
$videoWorkerUnderLoad = $videoWorkers[0]->updateHeartbeat(
|
|
new Percentage(85), // High CPU for video encoding
|
|
Byte::fromGigabytes(3), // High memory usage
|
|
2 // At capacity
|
|
);
|
|
|
|
$audioWorkerLightLoad = $audioWorkers[0]->updateHeartbeat(
|
|
new Percentage(30), // Moderate CPU
|
|
Byte::fromMegabytes(800),
|
|
3 // 3/8 jobs
|
|
);
|
|
|
|
$thumbnailWorkerIdle = $thumbnailWorkers[0]->updateHeartbeat(
|
|
new Percentage(5), // Very low CPU
|
|
Byte::fromMegabytes(200),
|
|
0 // No current jobs
|
|
);
|
|
|
|
// Verify load distribution logic
|
|
expect($videoWorkerUnderLoad->isAvailableForJobs())->toBeFalse(); // At capacity
|
|
expect($audioWorkerLightLoad->isAvailableForJobs())->toBeTrue();
|
|
expect($thumbnailWorkerIdle->isAvailableForJobs())->toBeTrue();
|
|
|
|
// Check load percentages
|
|
expect($videoWorkerUnderLoad->getLoadPercentage()->getValue())->toBe(100.0); // 2/2 jobs = 100%
|
|
expect($audioWorkerLightLoad->getLoadPercentage()->getValue())->toBe(37.5); // 3/8 = 37.5%
|
|
expect($thumbnailWorkerIdle->getLoadPercentage()->getValue())->toBe(5.0); // CPU load only
|
|
|
|
expect(count($allWorkers))->toBe(5);
|
|
});
|
|
|
|
it('prevents resource exhaustion through proper capability matching', function () {
|
|
// Worker without GPU capabilities trying to handle video processing
|
|
$cpuOnlyWorker = Worker::register(
|
|
hostname: 'cpu-server',
|
|
processId: 7001,
|
|
queues: [QueueName::fromString('video-processing')],
|
|
maxJobs: 10,
|
|
capabilities: ['cpu-encoding'] // Missing GPU capability
|
|
);
|
|
|
|
$gpuWorker = Worker::register(
|
|
hostname: 'gpu-server',
|
|
processId: 7002,
|
|
queues: [QueueName::fromString('video-processing')],
|
|
maxJobs: 2,
|
|
capabilities: ['gpu-acceleration', 'video-encode', 'h264']
|
|
);
|
|
|
|
// Job requiring GPU acceleration
|
|
$jobData = [
|
|
'required_capabilities' => ['gpu-acceleration', 'h264'],
|
|
'resource_requirements' => [
|
|
'gpu_memory' => '4GB',
|
|
'encoding_quality' => 'high',
|
|
],
|
|
];
|
|
|
|
// Mock worker scoring (would normally be done by JobDistributionService)
|
|
// CPU-only worker should get score 0 (missing required capability)
|
|
expect($cpuOnlyWorker->hasCapability('gpu-acceleration'))->toBeFalse();
|
|
expect($gpuWorker->hasCapability('gpu-acceleration'))->toBeTrue();
|
|
expect($gpuWorker->hasCapability('h264'))->toBeTrue();
|
|
});
|
|
});
|
|
|
|
describe('Financial Transaction Processing Scenario', function () {
|
|
it('ensures transaction consistency with distributed locking', function () {
|
|
// Scenario: Banking system processing concurrent transactions
|
|
|
|
$transactionWorkers = [
|
|
Worker::register(
|
|
hostname: 'transaction-server-1',
|
|
processId: 8001,
|
|
queues: [QueueName::fromString('transactions')],
|
|
maxJobs: 50, // High throughput for financial data
|
|
capabilities: ['payment-processing', 'fraud-detection', 'pci-compliant']
|
|
),
|
|
Worker::register(
|
|
hostname: 'transaction-server-2',
|
|
processId: 8002,
|
|
queues: [QueueName::fromString('transactions')],
|
|
maxJobs: 50,
|
|
capabilities: ['payment-processing', 'fraud-detection', 'pci-compliant']
|
|
),
|
|
];
|
|
|
|
// Simulate concurrent transaction processing
|
|
$accountId = 'account-12345';
|
|
$transactionLock = LockKey::forResource('account', $accountId);
|
|
|
|
// Mock lock acquisition for account processing
|
|
$lockStmt = mock(\PDOStatement::class);
|
|
$lockStmt->shouldReceive('execute')->andReturn(true);
|
|
$lockStmt->shouldReceive('rowCount')->andReturn(1);
|
|
|
|
$failLockStmt = mock(\PDOStatement::class);
|
|
$failLockStmt->shouldReceive('execute')->andThrow(
|
|
new \PDOException('Duplicate entry for key PRIMARY')
|
|
);
|
|
|
|
$this->connection->shouldReceive('prepare')->andReturn(
|
|
$lockStmt, // First worker gets lock
|
|
$failLockStmt // Second worker fails
|
|
);
|
|
|
|
// First worker should acquire lock successfully
|
|
$worker1 = $transactionWorkers[0];
|
|
$worker2 = $transactionWorkers[1];
|
|
|
|
$firstLockResult = $this->distributedLock->acquire(
|
|
$transactionLock,
|
|
$worker1->id,
|
|
Duration::fromMinutes(5)
|
|
);
|
|
|
|
$secondLockResult = $this->distributedLock->acquire(
|
|
$transactionLock,
|
|
$worker2->id,
|
|
Duration::fromMinutes(5)
|
|
);
|
|
|
|
expect($firstLockResult)->toBeTrue();
|
|
expect($secondLockResult)->toBeFalse(); // Should fail due to existing lock
|
|
});
|
|
|
|
it('handles high-frequency trading with minimal latency', function () {
|
|
// High-performance workers for trading operations
|
|
$tradingWorker = Worker::register(
|
|
hostname: 'trading-server-hft',
|
|
processId: 9001,
|
|
queues: [QueueName::fromString('high-frequency-trading')],
|
|
maxJobs: 1000, // Very high throughput
|
|
capabilities: ['ultra-low-latency', 'market-data', 'order-execution']
|
|
);
|
|
|
|
// Simulate high load but healthy performance
|
|
$performantWorker = $tradingWorker->updateHeartbeat(
|
|
new Percentage(60), // Moderate CPU despite high load
|
|
Byte::fromGigabytes(1.5), // Efficient memory usage
|
|
800 // High job count but within limits
|
|
);
|
|
|
|
expect($performantWorker->isHealthy())->toBeTrue();
|
|
expect($performantWorker->isAvailableForJobs())->toBeTrue(); // Still has capacity
|
|
expect($performantWorker->getLoadPercentage()->getValue())->toBe(80.0); // 800/1000 jobs
|
|
});
|
|
});
|
|
|
|
describe('Content Delivery Network Scenario', function () {
|
|
it('distributes cache warming jobs across geographic regions', function () {
|
|
// Workers in different geographic regions
|
|
$usEastWorkers = [
|
|
Worker::register(
|
|
hostname: 'cdn-us-east-1',
|
|
processId: 10001,
|
|
queues: [QueueName::fromString('cache-warming')],
|
|
maxJobs: 25,
|
|
capabilities: ['cdn-management', 'us-east-region', 'edge-caching']
|
|
),
|
|
Worker::register(
|
|
hostname: 'cdn-us-east-2',
|
|
processId: 10002,
|
|
queues: [QueueName::fromString('cache-warming')],
|
|
maxJobs: 25,
|
|
capabilities: ['cdn-management', 'us-east-region', 'edge-caching']
|
|
),
|
|
];
|
|
|
|
$europeWorkers = [
|
|
Worker::register(
|
|
hostname: 'cdn-eu-west-1',
|
|
processId: 11001,
|
|
queues: [QueueName::fromString('cache-warming')],
|
|
maxJobs: 20,
|
|
capabilities: ['cdn-management', 'eu-west-region', 'edge-caching']
|
|
),
|
|
];
|
|
|
|
$asiaWorkers = [
|
|
Worker::register(
|
|
hostname: 'cdn-asia-pacific-1',
|
|
processId: 12001,
|
|
queues: [QueueName::fromString('cache-warming')],
|
|
maxJobs: 15,
|
|
capabilities: ['cdn-management', 'asia-pacific-region', 'edge-caching']
|
|
),
|
|
];
|
|
|
|
$allCdnWorkers = array_merge($usEastWorkers, $europeWorkers, $asiaWorkers);
|
|
|
|
// Verify regional distribution
|
|
$usEastCount = count(array_filter(
|
|
$allCdnWorkers,
|
|
fn ($w) => $w->hasCapability('us-east-region')
|
|
));
|
|
$europeCount = count(array_filter(
|
|
$allCdnWorkers,
|
|
fn ($w) => $w->hasCapability('eu-west-region')
|
|
));
|
|
$asiaCount = count(array_filter(
|
|
$allCdnWorkers,
|
|
fn ($w) => $w->hasCapability('asia-pacific-region')
|
|
));
|
|
|
|
expect($usEastCount)->toBe(2);
|
|
expect($europeCount)->toBe(1);
|
|
expect($asiaCount)->toBe(1);
|
|
|
|
// Verify all workers can handle cache warming
|
|
foreach ($allCdnWorkers as $worker) {
|
|
expect($worker->hasCapability('cdn-management'))->toBeTrue();
|
|
expect($worker->hasCapability('edge-caching'))->toBeTrue();
|
|
}
|
|
});
|
|
|
|
it('handles regional worker failure with graceful degradation', function () {
|
|
// Scenario: Entire region goes offline, traffic redistributed
|
|
|
|
$primaryWorker = Worker::register(
|
|
hostname: 'cdn-primary',
|
|
processId: 13001,
|
|
queues: [QueueName::fromString('content-delivery')],
|
|
maxJobs: 100,
|
|
capabilities: ['primary-region', 'high-capacity']
|
|
);
|
|
|
|
$backupWorkers = [
|
|
Worker::register(
|
|
hostname: 'cdn-backup-1',
|
|
processId: 13002,
|
|
queues: [QueueName::fromString('content-delivery')],
|
|
maxJobs: 50,
|
|
capabilities: ['backup-region', 'medium-capacity']
|
|
),
|
|
Worker::register(
|
|
hostname: 'cdn-backup-2',
|
|
processId: 13003,
|
|
queues: [QueueName::fromString('content-delivery')],
|
|
maxJobs: 50,
|
|
capabilities: ['backup-region', 'medium-capacity']
|
|
),
|
|
];
|
|
|
|
// Simulate primary region failure
|
|
$failedPrimary = $primaryWorker->markInactive();
|
|
expect($failedPrimary->isAvailableForJobs())->toBeFalse();
|
|
|
|
// Backup workers should still be available
|
|
foreach ($backupWorkers as $backup) {
|
|
expect($backup->isAvailableForJobs())->toBeTrue();
|
|
}
|
|
|
|
// Total backup capacity should handle reduced load
|
|
$totalBackupCapacity = array_sum(array_map(fn ($w) => $w->maxJobs, $backupWorkers));
|
|
expect($totalBackupCapacity)->toBe(100); // Same as primary capacity
|
|
});
|
|
});
|
|
|
|
describe('Machine Learning Training Pipeline Scenario', function () {
|
|
it('manages resource-intensive ML training jobs efficiently', function () {
|
|
// Specialized workers for different ML tasks
|
|
$gpuTrainingWorker = Worker::register(
|
|
hostname: 'ml-gpu-cluster-1',
|
|
processId: 14001,
|
|
queues: [QueueName::fromString('ml-training')],
|
|
maxJobs: 1, // One intensive job at a time
|
|
capabilities: ['gpu-cluster', 'tensorflow', 'pytorch', 'cuda']
|
|
);
|
|
|
|
$dataPreprocessingWorkers = [
|
|
Worker::register(
|
|
hostname: 'ml-preprocessing-1',
|
|
processId: 14002,
|
|
queues: [QueueName::fromString('data-preprocessing')],
|
|
maxJobs: 10,
|
|
capabilities: ['data-cleaning', 'feature-engineering', 'pandas', 'numpy']
|
|
),
|
|
Worker::register(
|
|
hostname: 'ml-preprocessing-2',
|
|
processId: 14003,
|
|
queues: [QueueName::fromString('data-preprocessing')],
|
|
maxJobs: 10,
|
|
capabilities: ['data-cleaning', 'feature-engineering', 'pandas', 'numpy']
|
|
),
|
|
];
|
|
|
|
$inferenceWorkers = [
|
|
Worker::register(
|
|
hostname: 'ml-inference-1',
|
|
processId: 14004,
|
|
queues: [QueueName::fromString('ml-inference')],
|
|
maxJobs: 50, // High throughput for inference
|
|
capabilities: ['model-serving', 'tensorflow-lite', 'onnx']
|
|
),
|
|
];
|
|
|
|
// Simulate GPU worker under heavy load
|
|
$trainingWorkerLoaded = $gpuTrainingWorker->updateHeartbeat(
|
|
new Percentage(95), // High GPU utilization
|
|
Byte::fromGigabytes(15), // High memory for large models
|
|
1 // At capacity
|
|
);
|
|
|
|
// Preprocessing workers with moderate load
|
|
$preprocessingWorkerActive = $dataPreprocessingWorkers[0]->updateHeartbeat(
|
|
new Percentage(70), // Active data processing
|
|
Byte::fromGigabytes(4),
|
|
6 // 6/10 jobs
|
|
);
|
|
|
|
// Inference worker with light load
|
|
$inferenceWorkerIdle = $inferenceWorkers[0]->updateHeartbeat(
|
|
new Percentage(20), // Low CPU for inference
|
|
Byte::fromMegabytes(800),
|
|
5 // 5/50 jobs
|
|
);
|
|
|
|
// Verify resource allocation patterns
|
|
expect($trainingWorkerLoaded->isAvailableForJobs())->toBeFalse(); // At capacity
|
|
expect($preprocessingWorkerActive->isAvailableForJobs())->toBeTrue();
|
|
expect($inferenceWorkerIdle->isAvailableForJobs())->toBeTrue();
|
|
|
|
// Check load patterns match expected ML workloads
|
|
expect($trainingWorkerLoaded->getLoadPercentage()->getValue())->toBe(100.0); // At capacity
|
|
expect($preprocessingWorkerActive->getLoadPercentage()->getValue())->toBe(70.0); // CPU bound
|
|
expect($inferenceWorkerIdle->getLoadPercentage()->getValue())->toBe(20.0); // Light load
|
|
});
|
|
});
|
|
});
|