Michael Schiemer
fc3d7e6357
- Add comprehensive health check system with multiple endpoints - Add Prometheus metrics endpoint - Add production logging configurations (5 strategies) - Add complete deployment documentation suite: * QUICKSTART.md - 30-minute deployment guide * DEPLOYMENT_CHECKLIST.md - Printable verification checklist * DEPLOYMENT_WORKFLOW.md - Complete deployment lifecycle * PRODUCTION_DEPLOYMENT.md - Comprehensive technical reference * production-logging.md - Logging configuration guide * ANSIBLE_DEPLOYMENT.md - Infrastructure as Code automation * README.md - Navigation hub * DEPLOYMENT_SUMMARY.md - Executive summary - Add deployment scripts and automation - Add DEPLOYMENT_PLAN.md - Concrete plan for immediate deployment - Update README with production-ready features All production infrastructure is now complete and ready for deployment.
370 lines
14 KiB
PHP
370 lines
14 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use App\Framework\Database\EntityManagerInterface;
|
|
use App\Framework\DI\Container;
|
|
use App\Framework\Logging\Logger;
|
|
use App\Framework\Queue\Contracts\JobChainManagerInterface;
|
|
use App\Framework\Queue\Contracts\JobDependencyManagerInterface;
|
|
use App\Framework\Queue\Contracts\QueueInterface;
|
|
use App\Framework\Queue\Services\DependencyResolutionEngine;
|
|
use App\Framework\Queue\Services\JobChainExecutionCoordinator;
|
|
use App\Framework\Queue\Services\JobMetricsManager;
|
|
use App\Framework\Queue\ValueObjects\ChainExecutionMode;
|
|
use App\Framework\Queue\ValueObjects\JobChain;
|
|
use App\Framework\Queue\ValueObjects\JobDependency;
|
|
use App\Framework\Queue\ValueObjects\JobMetrics;
|
|
|
|
beforeEach(function () {
|
|
// Set up test container
|
|
$this->container = createTestContainer();
|
|
|
|
// Get services from container
|
|
$this->queue = $this->container->get(QueueInterface::class);
|
|
$this->dependencyManager = $this->container->get(JobDependencyManagerInterface::class);
|
|
$this->chainManager = $this->container->get(JobChainManagerInterface::class);
|
|
$this->metricsManager = $this->container->get(JobMetricsManager::class);
|
|
$this->resolutionEngine = $this->container->get(DependencyResolutionEngine::class);
|
|
$this->chainCoordinator = $this->container->get(JobChainExecutionCoordinator::class);
|
|
$this->entityManager = $this->container->get(EntityManagerInterface::class);
|
|
$this->logger = $this->container->get(Logger::class);
|
|
});
|
|
|
|
function createTestJob(string $id, string $data): object
|
|
{
|
|
return new class ($id, $data) {
|
|
public function __construct(
|
|
public readonly string $id,
|
|
public readonly string $data
|
|
) {
|
|
}
|
|
};
|
|
}
|
|
|
|
test('complete queue workflow with dependencies and metrics', function () {
|
|
// 1. Create test jobs
|
|
$job1 = createTestJob('job-1', 'Test Job 1');
|
|
$job2 = createTestJob('job-2', 'Test Job 2');
|
|
$job3 = createTestJob('job-3', 'Test Job 3');
|
|
|
|
// 2. Set up dependencies: job2 depends on job1, job3 depends on job2
|
|
$dependency1 = JobDependency::completion('job-2', 'job-1');
|
|
$dependency2 = JobDependency::success('job-3', 'job-2');
|
|
|
|
// Add dependencies
|
|
$this->dependencyManager->addDependency($dependency1);
|
|
$this->dependencyManager->addDependency($dependency2);
|
|
|
|
// 3. Add jobs to queue
|
|
$this->queue->push($job1);
|
|
$this->queue->push($job2);
|
|
$this->queue->push($job3);
|
|
|
|
// 4. Create and record metrics for job execution
|
|
$job1Metrics = new JobMetrics(
|
|
jobId: 'job-1',
|
|
queueName: 'default',
|
|
status: 'completed',
|
|
attempts: 1,
|
|
maxAttempts: 3,
|
|
executionTimeMs: 150.5,
|
|
memoryUsageBytes: 1024 * 1024,
|
|
errorMessage: null,
|
|
createdAt: date('Y-m-d H:i:s'),
|
|
startedAt: date('Y-m-d H:i:s'),
|
|
completedAt: date('Y-m-d H:i:s'),
|
|
failedAt: null,
|
|
metadata: ['test' => true]
|
|
);
|
|
|
|
$this->metricsManager->recordJobMetrics($job1Metrics);
|
|
|
|
// 5. Test dependency resolution
|
|
$readyJobs = $this->resolutionEngine->getJobsReadyForExecution();
|
|
expect($readyJobs)->toHaveCount(1)
|
|
->and($readyJobs[0])->toBe('job-1');
|
|
|
|
// 6. Mark job1 as completed and check dependencies
|
|
$this->dependencyManager->markJobCompleted('job-1');
|
|
|
|
$readyJobsAfterJob1 = $this->resolutionEngine->getJobsReadyForExecution();
|
|
expect($readyJobsAfterJob1)->toContain('job-2');
|
|
|
|
// 7. Test metrics retrieval
|
|
$retrievedMetrics = $this->metricsManager->getJobMetrics('job-1');
|
|
expect($retrievedMetrics)->not()->toBeNull()
|
|
->and($retrievedMetrics->jobId)->toBe('job-1')
|
|
->and($retrievedMetrics->status)->toBe('completed')
|
|
->and($retrievedMetrics->executionTimeMs)->toBe(150.5);
|
|
|
|
// 8. Test queue metrics calculation
|
|
$queueMetrics = $this->metricsManager->getQueueMetrics('default', '1 hour');
|
|
expect($queueMetrics->queueName)->toBe('default')
|
|
->and($queueMetrics->totalJobs)->toBeGreaterThan(0);
|
|
});
|
|
|
|
test('job chain execution with sequential mode', function () {
|
|
// 1. Create jobs for chain
|
|
$jobs = [
|
|
createTestJob('chain-job-1', 'Chain Job 1'),
|
|
createTestJob('chain-job-2', 'Chain Job 2'),
|
|
createTestJob('chain-job-3', 'Chain Job 3'),
|
|
];
|
|
|
|
// 2. Create job chain
|
|
$chain = JobChain::sequential('test-chain', ['chain-job-1', 'chain-job-2', 'chain-job-3']);
|
|
|
|
// 3. Add chain to manager
|
|
$this->chainManager->createChain($chain);
|
|
|
|
// 4. Execute chain
|
|
$this->chainCoordinator->executeChain('test-chain');
|
|
|
|
// 5. Verify chain was created
|
|
$retrievedChain = $this->chainManager->getChain('test-chain');
|
|
expect($retrievedChain)->not()->toBeNull()
|
|
->and($retrievedChain->name)->toBe('test-chain')
|
|
->and($retrievedChain->executionMode)->toBe(ChainExecutionMode::SEQUENTIAL)
|
|
->and($retrievedChain->jobIds)->toHaveCount(3);
|
|
});
|
|
|
|
test('job chain failure handling', function () {
|
|
// 1. Create jobs for chain with one that will fail
|
|
$jobs = [
|
|
createTestJob('fail-job-1', 'Job 1'),
|
|
createTestJob('fail-job-2', 'Job 2 (will fail)'),
|
|
createTestJob('fail-job-3', 'Job 3'),
|
|
];
|
|
|
|
// 2. Create job chain with stop on failure
|
|
$chain = JobChain::sequential('fail-chain', ['fail-job-1', 'fail-job-2', 'fail-job-3']);
|
|
$this->chainManager->createChain($chain);
|
|
|
|
// 3. Simulate job failure
|
|
$failureMetrics = new JobMetrics(
|
|
jobId: 'fail-job-2',
|
|
queueName: 'default',
|
|
status: 'failed',
|
|
attempts: 3,
|
|
maxAttempts: 3,
|
|
executionTimeMs: 50.0,
|
|
memoryUsageBytes: 512 * 1024,
|
|
errorMessage: 'Simulated failure',
|
|
createdAt: date('Y-m-d H:i:s'),
|
|
startedAt: date('Y-m-d H:i:s'),
|
|
completedAt: null,
|
|
failedAt: date('Y-m-d H:i:s'),
|
|
metadata: []
|
|
);
|
|
|
|
$this->metricsManager->recordJobMetrics($failureMetrics);
|
|
|
|
// 4. Test failure detection
|
|
$failedJobs = $this->metricsManager->getFailedJobs('default', '1 hour');
|
|
expect($failedJobs)->toHaveCount(1)
|
|
->and($failedJobs[0]->jobId)->toBe('fail-job-2')
|
|
->and($failedJobs[0]->status)->toBe('failed');
|
|
});
|
|
|
|
test('circular dependency detection', function () {
|
|
// 1. Create circular dependencies: A depends on B, B depends on C, C depends on A
|
|
$depA = JobDependency::completion('job-a', 'job-b');
|
|
$depB = JobDependency::completion('job-b', 'job-c');
|
|
$depC = JobDependency::completion('job-c', 'job-a');
|
|
|
|
// 2. Add dependencies
|
|
$this->dependencyManager->addDependency($depA);
|
|
$this->dependencyManager->addDependency($depB);
|
|
|
|
// 3. Adding the third dependency should throw an exception or be handled
|
|
expect(fn () => $this->dependencyManager->addDependency($depC))
|
|
->toThrow(\InvalidArgumentException::class);
|
|
});
|
|
|
|
test('conditional dependencies', function () {
|
|
// 1. Create jobs
|
|
$job1 = createTestJob('cond-job-1', 'Conditional Job 1');
|
|
$job2 = createTestJob('cond-job-2', 'Conditional Job 2');
|
|
|
|
// 2. Create success-based dependency
|
|
$successDep = JobDependency::success('cond-job-2', 'cond-job-1');
|
|
$this->dependencyManager->addDependency($successDep);
|
|
|
|
// 3. Test that job2 is not ready when job1 failed
|
|
$failureMetrics = new JobMetrics(
|
|
jobId: 'cond-job-1',
|
|
queueName: 'default',
|
|
status: 'failed',
|
|
attempts: 3,
|
|
maxAttempts: 3,
|
|
executionTimeMs: 100.0,
|
|
memoryUsageBytes: 1024,
|
|
errorMessage: 'Test failure',
|
|
createdAt: date('Y-m-d H:i:s'),
|
|
startedAt: date('Y-m-d H:i:s'),
|
|
completedAt: null,
|
|
failedAt: date('Y-m-d H:i:s'),
|
|
metadata: []
|
|
);
|
|
|
|
$this->metricsManager->recordJobMetrics($failureMetrics);
|
|
|
|
// 4. Check that dependent job is not ready
|
|
$readyJobs = $this->resolutionEngine->getJobsReadyForExecution();
|
|
expect($readyJobs)->not()->toContain('cond-job-2');
|
|
});
|
|
|
|
test('queue metrics calculation', function () {
|
|
// 1. Create multiple job metrics
|
|
$metrics = [
|
|
new JobMetrics(
|
|
jobId: 'metric-job-1',
|
|
queueName: 'metrics-queue',
|
|
status: 'completed',
|
|
attempts: 1,
|
|
maxAttempts: 3,
|
|
executionTimeMs: 100.0,
|
|
memoryUsageBytes: 1024 * 1024,
|
|
errorMessage: null,
|
|
createdAt: date('Y-m-d H:i:s'),
|
|
startedAt: date('Y-m-d H:i:s'),
|
|
completedAt: date('Y-m-d H:i:s'),
|
|
failedAt: null,
|
|
metadata: []
|
|
),
|
|
new JobMetrics(
|
|
jobId: 'metric-job-2',
|
|
queueName: 'metrics-queue',
|
|
status: 'completed',
|
|
attempts: 1,
|
|
maxAttempts: 3,
|
|
executionTimeMs: 200.0,
|
|
memoryUsageBytes: 2 * 1024 * 1024,
|
|
errorMessage: null,
|
|
createdAt: date('Y-m-d H:i:s'),
|
|
startedAt: date('Y-m-d H:i:s'),
|
|
completedAt: date('Y-m-d H:i:s'),
|
|
failedAt: null,
|
|
metadata: []
|
|
),
|
|
new JobMetrics(
|
|
jobId: 'metric-job-3',
|
|
queueName: 'metrics-queue',
|
|
status: 'failed',
|
|
attempts: 3,
|
|
maxAttempts: 3,
|
|
executionTimeMs: 50.0,
|
|
memoryUsageBytes: 512 * 1024,
|
|
errorMessage: 'Test failure',
|
|
createdAt: date('Y-m-d H:i:s'),
|
|
startedAt: date('Y-m-d H:i:s'),
|
|
completedAt: null,
|
|
failedAt: date('Y-m-d H:i:s'),
|
|
metadata: []
|
|
),
|
|
];
|
|
|
|
// 2. Record all metrics
|
|
foreach ($metrics as $metric) {
|
|
$this->metricsManager->recordJobMetrics($metric);
|
|
}
|
|
|
|
// 3. Calculate queue metrics
|
|
$queueMetrics = $this->metricsManager->getQueueMetrics('metrics-queue', '1 hour');
|
|
|
|
// 4. Verify calculations
|
|
expect($queueMetrics->queueName)->toBe('metrics-queue')
|
|
->and($queueMetrics->totalJobs)->toBe(3)
|
|
->and($queueMetrics->completedJobs)->toBe(2)
|
|
->and($queueMetrics->failedJobs)->toBe(1)
|
|
->and($queueMetrics->successRate->toFloat())->toBe(66.67);
|
|
});
|
|
|
|
test('dead letter queue functionality', function () {
|
|
// 1. Create a job that exceeds max attempts
|
|
$deadLetterMetrics = new JobMetrics(
|
|
jobId: 'dead-letter-job',
|
|
queueName: 'default',
|
|
status: 'failed',
|
|
attempts: 3,
|
|
maxAttempts: 3,
|
|
executionTimeMs: 25.0,
|
|
memoryUsageBytes: 256 * 1024,
|
|
errorMessage: 'Max attempts exceeded',
|
|
createdAt: date('Y-m-d H:i:s'),
|
|
startedAt: date('Y-m-d H:i:s'),
|
|
completedAt: null,
|
|
failedAt: date('Y-m-d H:i:s'),
|
|
metadata: ['dead_letter' => true]
|
|
);
|
|
|
|
// 2. Record metrics
|
|
$this->metricsManager->recordJobMetrics($deadLetterMetrics);
|
|
|
|
// 3. Verify dead letter detection
|
|
$failedJobs = $this->metricsManager->getFailedJobs('default', '1 hour');
|
|
$deadLetterJob = array_filter($failedJobs, fn ($job) => $job->jobId === 'dead-letter-job')[0] ?? null;
|
|
|
|
expect($deadLetterJob)->not()->toBeNull()
|
|
->and($deadLetterJob->attempts)->toBe(3)
|
|
->and($deadLetterJob->maxAttempts)->toBe(3)
|
|
->and($deadLetterJob->status)->toBe('failed');
|
|
});
|
|
|
|
test('system health monitoring', function () {
|
|
// 1. Create mixed job metrics for health calculation
|
|
$healthMetrics = [
|
|
// Healthy jobs
|
|
new JobMetrics('health-1', 'health-queue', 'completed', 1, 3, 50.0, 1024, null, date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), null, []),
|
|
new JobMetrics('health-2', 'health-queue', 'completed', 1, 3, 75.0, 1024, null, date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), null, []),
|
|
new JobMetrics('health-3', 'health-queue', 'completed', 1, 3, 100.0, 1024, null, date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), null, []),
|
|
// One failed job
|
|
new JobMetrics('health-4', 'health-queue', 'failed', 2, 3, 25.0, 1024, 'Health test failure', date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), null, date('Y-m-d H:i:s'), []),
|
|
];
|
|
|
|
// 2. Record all metrics
|
|
foreach ($healthMetrics as $metric) {
|
|
$this->metricsManager->recordJobMetrics($metric);
|
|
}
|
|
|
|
// 3. Get system overview
|
|
$overview = $this->metricsManager->getSystemOverview();
|
|
|
|
// 4. Verify system health calculation
|
|
expect($overview)->toHaveKey('system_health_score')
|
|
->and($overview['total_jobs'])->toBeGreaterThan(0)
|
|
->and($overview['overall_success_rate'])->toBeGreaterThan(0);
|
|
});
|
|
|
|
test('performance and throughput metrics', function () {
|
|
// 1. Create performance test metrics with varying execution times
|
|
$performanceMetrics = [
|
|
new JobMetrics('perf-1', 'perf-queue', 'completed', 1, 3, 50.0, 1024 * 1024, null, date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), null, []),
|
|
new JobMetrics('perf-2', 'perf-queue', 'completed', 1, 3, 150.0, 2 * 1024 * 1024, null, date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), null, []),
|
|
new JobMetrics('perf-3', 'perf-queue', 'completed', 1, 3, 300.0, 4 * 1024 * 1024, null, date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), date('Y-m-d H:i:s'), null, []),
|
|
];
|
|
|
|
// 2. Record performance metrics
|
|
foreach ($performanceMetrics as $metric) {
|
|
$this->metricsManager->recordJobMetrics($metric);
|
|
}
|
|
|
|
// 3. Get performance statistics
|
|
$performanceStats = $this->metricsManager->getPerformanceStats('perf-queue', '1 hour');
|
|
|
|
// 4. Verify performance calculations
|
|
expect($performanceStats)->toHaveKey('average_execution_time_ms')
|
|
->and($performanceStats['average_execution_time_ms'])->toBe(166.67)
|
|
->and($performanceStats)->toHaveKey('average_memory_usage_mb')
|
|
->and($performanceStats['total_jobs'])->toBe(3);
|
|
|
|
// 5. Get throughput statistics
|
|
$throughputStats = $this->metricsManager->getThroughputStats('perf-queue', '1 hour');
|
|
|
|
// 6. Verify throughput calculations
|
|
expect($throughputStats)->toHaveKey('total_completed')
|
|
->and($throughputStats['total_completed'])->toBe(3)
|
|
->and($throughputStats)->toHaveKey('average_throughput_per_hour');
|
|
});
|