Michael Schiemer
3b623e7afb
- Create AnsibleDeployStage using framework's Process module for secure command execution - Integrate AnsibleDeployStage into DeploymentPipelineCommands for production deployments - Add force_deploy flag support in Ansible playbook to override stale locks - Use PHP deployment module as orchestrator (php console.php deploy:production) - Fix ErrorAggregationInitializer to use Environment class instead of $_ENV superglobal Architecture: - BuildStage → AnsibleDeployStage → HealthCheckStage for production - Process module provides timeout, error handling, and output capture - Ansible playbook supports rollback via rollback-git-based.yml - Zero-downtime deployments with health checks
517 lines
17 KiB
PHP
517 lines
17 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use App\Framework\Core\ValueObjects\Version;
|
|
use App\Framework\Core\ValueObjects\Timestamp;
|
|
use App\Framework\MachineLearning\ModelManagement\DatabaseModelRegistry;
|
|
use App\Framework\MachineLearning\ModelManagement\DatabasePerformanceStorage;
|
|
use App\Framework\MachineLearning\ModelManagement\ModelPerformanceMonitor;
|
|
use App\Framework\MachineLearning\ModelManagement\NotificationAlertingService;
|
|
use App\Framework\MachineLearning\ModelManagement\MLConfig;
|
|
use App\Framework\MachineLearning\ModelManagement\ValueObjects\ModelMetadata;
|
|
use App\Framework\MachineLearning\ModelManagement\ValueObjects\ModelType;
|
|
use App\Framework\Database\ConnectionInterface;
|
|
use App\Framework\Database\ValueObjects\SqlQuery;
|
|
use App\Framework\Notification\NotificationDispatcher;
|
|
|
|
/**
|
|
* Integration Tests for ML Management System
|
|
*
|
|
* Tests the complete ML Management system including:
|
|
* - DatabaseModelRegistry
|
|
* - DatabasePerformanceStorage
|
|
* - ModelPerformanceMonitor
|
|
* - NotificationAlertingService
|
|
* - MLConfig
|
|
*/
|
|
|
|
describe('ML Management System Integration', function () {
|
|
|
|
beforeEach(function () {
|
|
// Get services from container
|
|
$this->connection = container()->get(ConnectionInterface::class);
|
|
$this->registry = container()->get(DatabaseModelRegistry::class);
|
|
$this->storage = container()->get(DatabasePerformanceStorage::class);
|
|
$this->config = container()->get(MLConfig::class);
|
|
$this->dispatcher = container()->get(NotificationDispatcher::class);
|
|
|
|
// Clean up test data
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_models WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_predictions WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_confidence_baselines WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
});
|
|
|
|
afterEach(function () {
|
|
// Clean up test data
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_models WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_predictions WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_confidence_baselines WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
});
|
|
|
|
test('can register a new model in database', function () {
|
|
$metadata = new ModelMetadata(
|
|
modelName: 'test-sentiment-analyzer',
|
|
modelType: ModelType::SUPERVISED,
|
|
version: new Version(1, 0, 0),
|
|
configuration: ['hidden_layers' => 3, 'learning_rate' => 0.001],
|
|
performanceMetrics: ['accuracy' => 0.95, 'precision' => 0.93],
|
|
createdAt: Timestamp::now(),
|
|
deployedAt: Timestamp::now(),
|
|
environment: 'production',
|
|
metadata: ['description' => 'Test sentiment analysis model']
|
|
);
|
|
|
|
$this->registry->register($metadata);
|
|
|
|
// Verify model was registered
|
|
$retrievedMetadata = $this->registry->get('test-sentiment-analyzer', new Version(1, 0, 0));
|
|
|
|
expect($retrievedMetadata)->not->toBeNull();
|
|
expect($retrievedMetadata->modelName)->toBe('test-sentiment-analyzer');
|
|
expect($retrievedMetadata->version->toString())->toBe('1.0.0');
|
|
expect($retrievedMetadata->modelType)->toBe(ModelType::SUPERVISED);
|
|
expect($retrievedMetadata->isDeployed())->toBeTrue();
|
|
expect($retrievedMetadata->environment)->toBe('production');
|
|
});
|
|
|
|
test('can update model deployment status', function () {
|
|
$metadata = new ModelMetadata(
|
|
modelName: 'test-recommender',
|
|
modelType: ModelType::SUPERVISED,
|
|
version: new Version(2, 1, 0),
|
|
configuration: ['features' => 100],
|
|
performanceMetrics: ['rmse' => 0.15],
|
|
createdAt: Timestamp::now(),
|
|
deployedAt: null,
|
|
environment: 'staging',
|
|
metadata: ['description' => 'Test recommendation model']
|
|
);
|
|
|
|
$this->registry->register($metadata);
|
|
|
|
// Update deployment status
|
|
$this->registry->updateDeploymentStatus('test-recommender', new Version(2, 1, 0), true);
|
|
|
|
// Verify update
|
|
$updated = $this->registry->get('test-recommender', new Version(2, 1, 0));
|
|
expect($updated->isDeployed())->toBeTrue();
|
|
});
|
|
|
|
test('can get all model names', function () {
|
|
// Register multiple models
|
|
$models = [
|
|
'test-classifier-1',
|
|
'test-classifier-2',
|
|
'test-regressor-1',
|
|
];
|
|
|
|
foreach ($models as $modelName) {
|
|
$metadata = new ModelMetadata(
|
|
modelName: $modelName,
|
|
modelType: ModelType::SUPERVISED,
|
|
version: new Version(1, 0, 0),
|
|
configuration: [],
|
|
performanceMetrics: [],
|
|
createdAt: Timestamp::now(),
|
|
deployedAt: null,
|
|
environment: 'development'
|
|
);
|
|
$this->registry->register($metadata);
|
|
}
|
|
|
|
$allNames = $this->registry->getAllModelNames();
|
|
|
|
foreach ($models as $expectedName) {
|
|
expect($allNames)->toContain($expectedName);
|
|
}
|
|
});
|
|
|
|
test('can store prediction records', function () {
|
|
$predictionRecord = [
|
|
'model_name' => 'test-predictor',
|
|
'version' => '1.0.0',
|
|
'prediction' => ['class' => 'positive', 'probability' => 0.85],
|
|
'actual' => ['class' => 'positive'],
|
|
'confidence' => 0.85,
|
|
'features' => ['text_length' => 150, 'sentiment_score' => 0.7],
|
|
'timestamp' => Timestamp::now(),
|
|
'is_correct' => true,
|
|
];
|
|
|
|
$this->storage->storePrediction($predictionRecord);
|
|
|
|
// Verify prediction was stored by getting recent predictions
|
|
$recentPredictions = $this->storage->getRecentPredictions(
|
|
'test-predictor',
|
|
new Version(1, 0, 0),
|
|
100
|
|
);
|
|
|
|
expect($recentPredictions)->toHaveCount(1);
|
|
expect($recentPredictions[0]['model_name'])->toBe('test-predictor');
|
|
expect($recentPredictions[0]['confidence'])->toBe(0.85);
|
|
});
|
|
|
|
test('can calculate accuracy from predictions', function () {
|
|
$modelName = 'test-accuracy-model';
|
|
$version = new Version(1, 0, 0);
|
|
|
|
// Store multiple predictions
|
|
$predictions = [
|
|
['prediction' => ['class' => 'A'], 'actual' => ['class' => 'A'], 'confidence' => 0.9, 'is_correct' => true],
|
|
['prediction' => ['class' => 'B'], 'actual' => ['class' => 'B'], 'confidence' => 0.85, 'is_correct' => true],
|
|
['prediction' => ['class' => 'A'], 'actual' => ['class' => 'B'], 'confidence' => 0.6, 'is_correct' => false],
|
|
['prediction' => ['class' => 'C'], 'actual' => ['class' => 'C'], 'confidence' => 0.95, 'is_correct' => true],
|
|
];
|
|
|
|
foreach ($predictions as $pred) {
|
|
$record = [
|
|
'model_name' => $modelName,
|
|
'version' => $version->toString(),
|
|
'prediction' => $pred['prediction'],
|
|
'actual' => $pred['actual'],
|
|
'confidence' => $pred['confidence'],
|
|
'features' => [],
|
|
'timestamp' => Timestamp::now(),
|
|
'is_correct' => $pred['is_correct'],
|
|
];
|
|
$this->storage->storePrediction($record);
|
|
}
|
|
|
|
// Calculate accuracy (should be 3/4 = 0.75)
|
|
$accuracy = $this->storage->calculateAccuracy($modelName, $version, 100);
|
|
|
|
expect($accuracy)->toBe(0.75);
|
|
});
|
|
|
|
test('can store and retrieve confidence baseline', function () {
|
|
$modelName = 'test-baseline-model';
|
|
$version = new Version(1, 2, 3);
|
|
|
|
$this->storage->storeConfidenceBaseline(
|
|
$modelName,
|
|
$version,
|
|
avgConfidence: 0.82,
|
|
stdDevConfidence: 0.12
|
|
);
|
|
|
|
$baseline = $this->storage->getConfidenceBaseline($modelName, $version);
|
|
|
|
expect($baseline)->not->toBeNull();
|
|
expect($baseline['avg_confidence'])->toBe(0.82);
|
|
expect($baseline['std_dev_confidence'])->toBe(0.12);
|
|
});
|
|
|
|
test('can update confidence baseline (upsert)', function () {
|
|
$modelName = 'test-upsert-model';
|
|
$version = new Version(1, 0, 0);
|
|
|
|
// Initial insert
|
|
$this->storage->storeConfidenceBaseline($modelName, $version, 0.80, 0.10);
|
|
|
|
// Update (upsert)
|
|
$this->storage->storeConfidenceBaseline($modelName, $version, 0.85, 0.08);
|
|
|
|
$baseline = $this->storage->getConfidenceBaseline($modelName, $version);
|
|
|
|
expect($baseline['avg_confidence'])->toBe(0.85);
|
|
expect($baseline['std_dev_confidence'])->toBe(0.08);
|
|
});
|
|
});
|
|
|
|
describe('Model Performance Monitor Integration', function () {
|
|
|
|
beforeEach(function () {
|
|
$this->connection = container()->get(ConnectionInterface::class);
|
|
$this->registry = container()->get(DatabaseModelRegistry::class);
|
|
$this->storage = container()->get(DatabasePerformanceStorage::class);
|
|
$this->config = MLConfig::testing(); // Use testing config
|
|
$this->alerting = new NotificationAlertingService(
|
|
container()->get(NotificationDispatcher::class),
|
|
$this->config
|
|
);
|
|
|
|
$this->monitor = new ModelPerformanceMonitor(
|
|
$this->registry,
|
|
$this->storage,
|
|
$this->alerting,
|
|
$this->config
|
|
);
|
|
|
|
// Clean up
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_models WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_predictions WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_confidence_baselines WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
});
|
|
|
|
afterEach(function () {
|
|
// Clean up
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_models WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_predictions WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
$this->connection->execute(
|
|
SqlQuery::create(
|
|
'DELETE FROM ml_confidence_baselines WHERE model_name LIKE ?',
|
|
['test-%']
|
|
)
|
|
);
|
|
});
|
|
|
|
test('can track prediction with performance monitoring', function () {
|
|
$modelName = 'test-tracking-model';
|
|
$version = new Version(1, 0, 0);
|
|
|
|
// Register model
|
|
$metadata = new ModelMetadata(
|
|
modelName: $modelName,
|
|
modelType: ModelType::SUPERVISED,
|
|
version: $version,
|
|
configuration: [],
|
|
performanceMetrics: ['baseline_accuracy' => 0.90],
|
|
createdAt: Timestamp::now(),
|
|
deployedAt: Timestamp::now(),
|
|
environment: 'production'
|
|
);
|
|
$this->registry->register($metadata);
|
|
|
|
// Track prediction
|
|
$this->monitor->trackPrediction(
|
|
$modelName,
|
|
$version,
|
|
prediction: ['class' => 'spam'],
|
|
confidence: 0.92,
|
|
features: ['word_count' => 50],
|
|
actual: ['class' => 'spam']
|
|
);
|
|
|
|
// Verify prediction was stored
|
|
$predictions = $this->storage->getRecentPredictions($modelName, $version, 10);
|
|
|
|
expect($predictions)->toHaveCount(1);
|
|
expect($predictions[0]['confidence'])->toBe(0.92);
|
|
});
|
|
|
|
test('can detect low confidence', function () {
|
|
$modelName = 'test-low-confidence-model';
|
|
$version = new Version(1, 0, 0);
|
|
|
|
// Store baseline with high confidence
|
|
$this->storage->storeConfidenceBaseline($modelName, $version, 0.85, 0.05);
|
|
|
|
// Store predictions with low confidence
|
|
for ($i = 0; $i < 50; $i++) {
|
|
$this->storage->storePrediction([
|
|
'model_name' => $modelName,
|
|
'version' => $version->toString(),
|
|
'prediction' => ['value' => $i],
|
|
'actual' => ['value' => $i],
|
|
'confidence' => 0.55, // Low confidence
|
|
'features' => [],
|
|
'timestamp' => Timestamp::now(),
|
|
'is_correct' => true,
|
|
]);
|
|
}
|
|
|
|
// Check for low confidence
|
|
$hasLowConfidence = $this->monitor->hasLowConfidence($modelName, $version);
|
|
|
|
expect($hasLowConfidence)->toBeTrue();
|
|
});
|
|
});
|
|
|
|
describe('Notification Integration', function () {
|
|
|
|
beforeEach(function () {
|
|
$this->dispatcher = container()->get(NotificationDispatcher::class);
|
|
$this->config = MLConfig::development();
|
|
$this->alerting = new NotificationAlertingService(
|
|
$this->dispatcher,
|
|
$this->config,
|
|
'test-admin'
|
|
);
|
|
});
|
|
|
|
test('can send generic alert', function () {
|
|
// This should not throw
|
|
$this->alerting->sendAlert(
|
|
'warning',
|
|
'Test Alert',
|
|
'This is a test alert message',
|
|
['test_data' => 'value']
|
|
);
|
|
|
|
expect(true)->toBeTrue();
|
|
});
|
|
|
|
test('can send drift detected alert', function () {
|
|
$this->alerting->alertDriftDetected(
|
|
'test-model',
|
|
new Version(1, 0, 0),
|
|
0.25
|
|
);
|
|
|
|
expect(true)->toBeTrue();
|
|
});
|
|
|
|
test('can send performance degradation alert', function () {
|
|
$this->alerting->alertPerformanceDegradation(
|
|
'test-model',
|
|
new Version(1, 0, 0),
|
|
currentAccuracy: 0.70,
|
|
baselineAccuracy: 0.90
|
|
);
|
|
|
|
expect(true)->toBeTrue();
|
|
});
|
|
|
|
test('can send low confidence alert', function () {
|
|
$this->alerting->alertLowConfidence(
|
|
'test-model',
|
|
new Version(1, 0, 0),
|
|
0.55
|
|
);
|
|
|
|
expect(true)->toBeTrue();
|
|
});
|
|
|
|
test('can send model deployed alert', function () {
|
|
$this->alerting->alertModelDeployed(
|
|
'test-model',
|
|
new Version(2, 0, 0),
|
|
'production'
|
|
);
|
|
|
|
expect(true)->toBeTrue();
|
|
});
|
|
|
|
test('respects monitoring disabled config', function () {
|
|
$config = new MLConfig(monitoringEnabled: false);
|
|
$alerting = new NotificationAlertingService(
|
|
$this->dispatcher,
|
|
$config,
|
|
'test-admin'
|
|
);
|
|
|
|
// Should not throw even with monitoring disabled
|
|
$alerting->alertDriftDetected(
|
|
'test-model',
|
|
new Version(1, 0, 0),
|
|
0.25
|
|
);
|
|
|
|
expect(true)->toBeTrue();
|
|
});
|
|
});
|
|
|
|
describe('MLConfig Integration', function () {
|
|
|
|
test('can create config from environment', function () {
|
|
$config = MLConfig::fromEnvironment();
|
|
|
|
expect($config)->toBeInstanceOf(MLConfig::class);
|
|
expect($config->monitoringEnabled)->toBeTrue();
|
|
expect($config->driftThreshold)->toBeGreaterThan(0);
|
|
});
|
|
|
|
test('production config has strict thresholds', function () {
|
|
$config = MLConfig::production();
|
|
|
|
expect($config->monitoringEnabled)->toBeTrue();
|
|
expect($config->autoTuningEnabled)->toBeFalse();
|
|
expect($config->driftThreshold)->toBe(0.15);
|
|
expect($config->confidenceAlertThreshold)->toBe(0.65);
|
|
});
|
|
|
|
test('development config has relaxed thresholds', function () {
|
|
$config = MLConfig::development();
|
|
|
|
expect($config->monitoringEnabled)->toBeTrue();
|
|
expect($config->autoTuningEnabled)->toBeTrue();
|
|
expect($config->driftThreshold)->toBe(0.25);
|
|
});
|
|
|
|
test('testing config has very relaxed thresholds', function () {
|
|
$config = MLConfig::testing();
|
|
|
|
expect($config->monitoringEnabled)->toBeFalse();
|
|
expect($config->autoTuningEnabled)->toBeTrue();
|
|
expect($config->driftThreshold)->toBe(0.50);
|
|
});
|
|
|
|
test('can detect drift using config threshold', function () {
|
|
$config = MLConfig::production();
|
|
|
|
expect($config->isDriftDetected(0.10))->toBeFalse(); // Below threshold
|
|
expect($config->isDriftDetected(0.20))->toBeTrue(); // Above threshold
|
|
});
|
|
|
|
test('can detect low confidence using config threshold', function () {
|
|
$config = MLConfig::production();
|
|
|
|
expect($config->isLowConfidence(0.70))->toBeFalse(); // Above threshold
|
|
expect($config->isLowConfidence(0.60))->toBeTrue(); // Below threshold
|
|
});
|
|
|
|
test('can detect low accuracy using config threshold', function () {
|
|
$config = MLConfig::production();
|
|
|
|
expect($config->isLowAccuracy(0.80))->toBeFalse(); // Above threshold
|
|
expect($config->isLowAccuracy(0.70))->toBeTrue(); // Below threshold
|
|
});
|
|
});
|