Michael Schiemer
3b623e7afb
- Create AnsibleDeployStage using framework's Process module for secure command execution - Integrate AnsibleDeployStage into DeploymentPipelineCommands for production deployments - Add force_deploy flag support in Ansible playbook to override stale locks - Use PHP deployment module as orchestrator (php console.php deploy:production) - Fix ErrorAggregationInitializer to use Environment class instead of $_ENV superglobal Architecture: - BuildStage → AnsibleDeployStage → HealthCheckStage for production - Process module provides timeout, error handling, and output capture - Ansible playbook supports rollback via rollback-git-based.yml - Zero-downtime deployments with health checks
435 lines
19 KiB
PHP
435 lines
19 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
/**
|
|
* ML Monitoring Dashboard Data Collection Test
|
|
*
|
|
* Demonstrates comprehensive monitoring data collection for ML systems:
|
|
* 1. Model performance metrics (accuracy, precision, recall, F1)
|
|
* 2. Prediction distribution and confidence histograms
|
|
* 3. Model version comparison and drift detection
|
|
* 4. A/B test status and progress
|
|
* 5. System health indicators
|
|
* 6. Performance degradation alerts
|
|
*/
|
|
|
|
require_once __DIR__ . '/../../vendor/autoload.php';
|
|
|
|
use App\Framework\MachineLearning\ModelManagement\InMemoryModelRegistry;
|
|
use App\Framework\MachineLearning\ModelManagement\ModelPerformanceMonitor;
|
|
use App\Framework\MachineLearning\ModelManagement\InMemoryPerformanceStorage;
|
|
use App\Framework\MachineLearning\ModelManagement\NullAlertingService;
|
|
use App\Framework\MachineLearning\ModelManagement\ABTestingService;
|
|
use App\Framework\MachineLearning\ModelManagement\AutoTuningEngine;
|
|
use App\Framework\MachineLearning\ModelManagement\ValueObjects\ModelMetadata;
|
|
use App\Framework\MachineLearning\ModelManagement\ValueObjects\ModelType;
|
|
use App\Framework\MachineLearning\ModelManagement\ValueObjects\ABTestConfig;
|
|
use App\Framework\Queue\MachineLearning\QueueAnomalyModelAdapter;
|
|
use App\Framework\Queue\MachineLearning\JobAnomalyDetector;
|
|
use App\Framework\Waf\MachineLearning\WafBehavioralModelAdapter;
|
|
use App\Framework\Waf\MachineLearning\BehaviorAnomalyDetector;
|
|
use App\Framework\Core\ValueObjects\Version;
|
|
use App\Framework\Core\ValueObjects\Timestamp;
|
|
use App\Framework\Core\ValueObjects\Duration;
|
|
use App\Framework\Core\ValueObjects\Score;
|
|
use App\Framework\Random\SecureRandomGenerator;
|
|
|
|
echo "=== ML Monitoring Dashboard Data Collection Test ===\n\n";
|
|
|
|
try {
|
|
// ========================================================================
|
|
// Setup: Initialize infrastructure
|
|
// ========================================================================
|
|
echo "1. Initializing ML Monitoring Infrastructure...\n";
|
|
|
|
$registry = new InMemoryModelRegistry();
|
|
$storage = new InMemoryPerformanceStorage();
|
|
$alerting = new NullAlertingService();
|
|
$performanceMonitor = new ModelPerformanceMonitor($registry, $storage, $alerting);
|
|
$random = new SecureRandomGenerator();
|
|
$abTesting = new ABTestingService($random, $registry);
|
|
$autoTuning = new AutoTuningEngine($performanceMonitor, $registry, $storage);
|
|
|
|
echo " ✓ Infrastructure initialized\n\n";
|
|
|
|
// ========================================================================
|
|
// Setup: Register multiple models with different types
|
|
// ========================================================================
|
|
echo "2. Registering multiple ML models...\n";
|
|
|
|
$models = [
|
|
'queue-anomaly' => [
|
|
'type' => ModelType::UNSUPERVISED,
|
|
'version' => Version::fromString('1.0.0'),
|
|
'config' => ['threshold' => 0.4, 'z_score_threshold' => 3.0]
|
|
],
|
|
'waf-behavioral' => [
|
|
'type' => ModelType::UNSUPERVISED,
|
|
'version' => Version::fromString('1.2.0'),
|
|
'config' => ['threshold' => 0.5, 'z_score_threshold' => 2.5]
|
|
],
|
|
'fraud-detector' => [
|
|
'type' => ModelType::SUPERVISED,
|
|
'version' => Version::fromString('2.0.0'),
|
|
'config' => ['threshold' => 0.7, 'algorithm' => 'xgboost']
|
|
],
|
|
'spam-classifier' => [
|
|
'type' => ModelType::SUPERVISED,
|
|
'version' => Version::fromString('1.5.0'),
|
|
'config' => ['threshold' => 0.6, 'algorithm' => 'naive_bayes']
|
|
],
|
|
];
|
|
|
|
foreach ($models as $modelName => $info) {
|
|
$metadata = new ModelMetadata(
|
|
modelName: $modelName,
|
|
modelType: $info['type'],
|
|
version: $info['version'],
|
|
configuration: $info['config'],
|
|
createdAt: Timestamp::now()
|
|
);
|
|
$registry->register($metadata);
|
|
echo " ✓ Registered: {$modelName} v{$info['version']->toString()} ({$info['type']->value})\n";
|
|
}
|
|
echo "\n";
|
|
|
|
// ========================================================================
|
|
// Setup: Simulate prediction data for all models
|
|
// ========================================================================
|
|
echo "3. Simulating prediction data...\n";
|
|
|
|
$timestamp = Timestamp::now();
|
|
|
|
// Queue Anomaly: 95% accuracy
|
|
$queuePredictions = [
|
|
...array_fill(0, 95, ['confidence' => 0.85, 'actual' => true, 'prediction' => true]),
|
|
...array_fill(0, 5, ['confidence' => 0.45, 'actual' => false, 'prediction' => true]),
|
|
];
|
|
|
|
foreach ($queuePredictions as $pred) {
|
|
$storage->storePrediction([
|
|
'model_name' => 'queue-anomaly',
|
|
'version' => '1.0.0',
|
|
'prediction' => $pred['prediction'],
|
|
'actual' => $pred['actual'],
|
|
'confidence' => $pred['confidence'],
|
|
'features' => [],
|
|
'timestamp' => $timestamp->toDateTime(),
|
|
'is_correct' => $pred['prediction'] === $pred['actual'],
|
|
]);
|
|
}
|
|
|
|
// WAF Behavioral: 88% accuracy
|
|
$wafPredictions = [
|
|
...array_fill(0, 88, ['confidence' => 0.9, 'actual' => true, 'prediction' => true]),
|
|
...array_fill(0, 12, ['confidence' => 0.55, 'actual' => false, 'prediction' => true]),
|
|
];
|
|
|
|
foreach ($wafPredictions as $pred) {
|
|
$storage->storePrediction([
|
|
'model_name' => 'waf-behavioral',
|
|
'version' => '1.2.0',
|
|
'prediction' => $pred['prediction'],
|
|
'actual' => $pred['actual'],
|
|
'confidence' => $pred['confidence'],
|
|
'features' => [],
|
|
'timestamp' => $timestamp->toDateTime(),
|
|
'is_correct' => $pred['prediction'] === $pred['actual'],
|
|
]);
|
|
}
|
|
|
|
// Fraud Detector: 92% accuracy
|
|
$fraudPredictions = [
|
|
...array_fill(0, 92, ['confidence' => 0.95, 'actual' => true, 'prediction' => true]),
|
|
...array_fill(0, 8, ['confidence' => 0.6, 'actual' => false, 'prediction' => true]),
|
|
];
|
|
|
|
foreach ($fraudPredictions as $pred) {
|
|
$storage->storePrediction([
|
|
'model_name' => 'fraud-detector',
|
|
'version' => '2.0.0',
|
|
'prediction' => $pred['prediction'],
|
|
'actual' => $pred['actual'],
|
|
'confidence' => $pred['confidence'],
|
|
'features' => [],
|
|
'timestamp' => $timestamp->toDateTime(),
|
|
'is_correct' => $pred['prediction'] === $pred['actual'],
|
|
]);
|
|
}
|
|
|
|
// Spam Classifier: 78% accuracy (degraded)
|
|
$spamPredictions = [
|
|
...array_fill(0, 78, ['confidence' => 0.7, 'actual' => true, 'prediction' => true]),
|
|
...array_fill(0, 22, ['confidence' => 0.65, 'actual' => false, 'prediction' => true]),
|
|
];
|
|
|
|
foreach ($spamPredictions as $pred) {
|
|
$storage->storePrediction([
|
|
'model_name' => 'spam-classifier',
|
|
'version' => '1.5.0',
|
|
'prediction' => $pred['prediction'],
|
|
'actual' => $pred['actual'],
|
|
'confidence' => $pred['confidence'],
|
|
'features' => [],
|
|
'timestamp' => $timestamp->toDateTime(),
|
|
'is_correct' => $pred['prediction'] === $pred['actual'],
|
|
]);
|
|
}
|
|
|
|
echo " ✓ Simulated 400 total predictions across 4 models\n\n";
|
|
|
|
// ========================================================================
|
|
// Dashboard Data 1: Model Performance Overview
|
|
// ========================================================================
|
|
echo "4. Collecting Model Performance Overview...\n";
|
|
|
|
$performanceOverview = [];
|
|
|
|
foreach (array_keys($models) as $modelName) {
|
|
$metadata = $registry->get($modelName, $models[$modelName]['version']);
|
|
$metrics = $performanceMonitor->getCurrentMetrics($modelName, $models[$modelName]['version']);
|
|
|
|
$performanceOverview[$modelName] = [
|
|
'version' => $models[$modelName]['version']->toString(),
|
|
'type' => $models[$modelName]['type']->value,
|
|
'accuracy' => $metrics['accuracy'],
|
|
'precision' => $metrics['precision'] ?? 0.0,
|
|
'recall' => $metrics['recall'] ?? 0.0,
|
|
'f1_score' => $metrics['f1_score'] ?? 0.0,
|
|
'total_predictions' => $metrics['total_predictions'],
|
|
'average_confidence' => $metrics['average_confidence'] ?? 0.0,
|
|
'threshold' => $models[$modelName]['config']['threshold'],
|
|
'status' => $metrics['accuracy'] >= 0.85 ? 'healthy' : 'degraded'
|
|
];
|
|
}
|
|
|
|
echo " → Performance Overview:\n";
|
|
foreach ($performanceOverview as $modelName => $data) {
|
|
echo " {$modelName}:\n";
|
|
echo " Accuracy: " . sprintf("%.1f%%", $data['accuracy'] * 100) . "\n";
|
|
echo " Precision: " . sprintf("%.1f%%", $data['precision'] * 100) . "\n";
|
|
echo " Recall: " . sprintf("%.1f%%", $data['recall'] * 100) . "\n";
|
|
echo " F1-Score: " . sprintf("%.1f%%", $data['f1_score'] * 100) . "\n";
|
|
echo " Predictions: {$data['total_predictions']}\n";
|
|
echo " Status: {$data['status']}\n";
|
|
}
|
|
echo "\n";
|
|
|
|
// ========================================================================
|
|
// Dashboard Data 2: Performance Degradation Alerts
|
|
// ========================================================================
|
|
echo "5. Checking Performance Degradation Alerts...\n";
|
|
|
|
$degradationAlerts = [];
|
|
|
|
foreach (array_keys($models) as $modelName) {
|
|
$metrics = $performanceMonitor->getCurrentMetrics($modelName, $models[$modelName]['version']);
|
|
|
|
if ($metrics['accuracy'] < 0.85) {
|
|
$degradationAlerts[] = [
|
|
'model_name' => $modelName,
|
|
'version' => $models[$modelName]['version']->toString(),
|
|
'current_accuracy' => $metrics['accuracy'],
|
|
'threshold' => 0.85,
|
|
'severity' => $metrics['accuracy'] < 0.7 ? 'critical' : 'warning',
|
|
'recommendation' => 'Consider retraining or rolling back to previous version'
|
|
];
|
|
}
|
|
}
|
|
|
|
echo " → Degradation Alerts: " . count($degradationAlerts) . " alert(s)\n";
|
|
foreach ($degradationAlerts as $alert) {
|
|
echo " [{$alert['severity']}] {$alert['model_name']} v{$alert['version']}\n";
|
|
echo " Accuracy: " . sprintf("%.1f%%", $alert['current_accuracy'] * 100) . " (threshold: " . sprintf("%.0f%%", $alert['threshold'] * 100) . ")\n";
|
|
echo " Recommendation: {$alert['recommendation']}\n";
|
|
}
|
|
echo "\n";
|
|
|
|
// ========================================================================
|
|
// Dashboard Data 3: Confusion Matrix Breakdown
|
|
// ========================================================================
|
|
echo "6. Collecting Confusion Matrix Data...\n";
|
|
|
|
$confusionMatrices = [];
|
|
|
|
foreach (array_keys($models) as $modelName) {
|
|
$metrics = $performanceMonitor->getCurrentMetrics($modelName, $models[$modelName]['version']);
|
|
|
|
if (isset($metrics['confusion_matrix'])) {
|
|
$confusionMatrices[$modelName] = [
|
|
'true_positive' => $metrics['confusion_matrix']['true_positive'],
|
|
'true_negative' => $metrics['confusion_matrix']['true_negative'],
|
|
'false_positive' => $metrics['confusion_matrix']['false_positive'],
|
|
'false_negative' => $metrics['confusion_matrix']['false_negative'],
|
|
'total' => $metrics['total_predictions'],
|
|
'false_positive_rate' => $metrics['confusion_matrix']['false_positive'] / $metrics['total_predictions'],
|
|
'false_negative_rate' => $metrics['confusion_matrix']['false_negative'] / $metrics['total_predictions'],
|
|
];
|
|
}
|
|
}
|
|
|
|
echo " → Confusion Matrices:\n";
|
|
foreach ($confusionMatrices as $modelName => $matrix) {
|
|
echo " {$modelName}:\n";
|
|
echo " TP: {$matrix['true_positive']}, TN: {$matrix['true_negative']}\n";
|
|
echo " FP: {$matrix['false_positive']}, FN: {$matrix['false_negative']}\n";
|
|
echo " FP Rate: " . sprintf("%.1f%%", $matrix['false_positive_rate'] * 100) . "\n";
|
|
echo " FN Rate: " . sprintf("%.1f%%", $matrix['false_negative_rate'] * 100) . "\n";
|
|
}
|
|
echo "\n";
|
|
|
|
// ========================================================================
|
|
// Dashboard Data 4: Model Registry Summary
|
|
// ========================================================================
|
|
echo "7. Collecting Model Registry Summary...\n";
|
|
|
|
$registrySummary = [
|
|
'total_models' => $registry->getTotalCount(),
|
|
'total_model_types' => count($registry->getAllModelNames()),
|
|
'models_by_type' => [
|
|
'supervised' => 0,
|
|
'unsupervised' => 0,
|
|
'reinforcement' => 0
|
|
],
|
|
'average_predictions_per_model' => 0
|
|
];
|
|
|
|
$totalPredictions = 0;
|
|
|
|
foreach (array_keys($models) as $modelName) {
|
|
$metadata = $registry->get($modelName, $models[$modelName]['version']);
|
|
$registrySummary['models_by_type'][$metadata->modelType->value]++;
|
|
|
|
$metrics = $performanceMonitor->getCurrentMetrics($modelName, $models[$modelName]['version']);
|
|
$totalPredictions += $metrics['total_predictions'];
|
|
}
|
|
|
|
$registrySummary['average_predictions_per_model'] = $totalPredictions / $registrySummary['total_model_types'];
|
|
|
|
echo " → Registry Summary:\n";
|
|
echo " Total Models: {$registrySummary['total_models']}\n";
|
|
echo " Model Types: {$registrySummary['total_model_types']}\n";
|
|
echo " Supervised: {$registrySummary['models_by_type']['supervised']}\n";
|
|
echo " Unsupervised: {$registrySummary['models_by_type']['unsupervised']}\n";
|
|
echo " Avg Predictions/Model: " . sprintf("%.0f", $registrySummary['average_predictions_per_model']) . "\n\n";
|
|
|
|
// ========================================================================
|
|
// Dashboard Data 5: System Health Indicators
|
|
// ========================================================================
|
|
echo "8. Collecting System Health Indicators...\n";
|
|
|
|
$healthIndicators = [
|
|
'overall_status' => 'healthy',
|
|
'healthy_models' => 0,
|
|
'degraded_models' => 0,
|
|
'average_accuracy' => 0.0,
|
|
'lowest_accuracy' => 1.0,
|
|
'highest_accuracy' => 0.0,
|
|
'total_predictions' => $totalPredictions,
|
|
'models_below_threshold' => []
|
|
];
|
|
|
|
$totalAccuracy = 0.0;
|
|
|
|
foreach (array_keys($models) as $modelName) {
|
|
$metrics = $performanceMonitor->getCurrentMetrics($modelName, $models[$modelName]['version']);
|
|
|
|
if ($metrics['accuracy'] >= 0.85) {
|
|
$healthIndicators['healthy_models']++;
|
|
} else {
|
|
$healthIndicators['degraded_models']++;
|
|
$healthIndicators['models_below_threshold'][] = $modelName;
|
|
}
|
|
|
|
$totalAccuracy += $metrics['accuracy'];
|
|
|
|
if ($metrics['accuracy'] < $healthIndicators['lowest_accuracy']) {
|
|
$healthIndicators['lowest_accuracy'] = $metrics['accuracy'];
|
|
}
|
|
|
|
if ($metrics['accuracy'] > $healthIndicators['highest_accuracy']) {
|
|
$healthIndicators['highest_accuracy'] = $metrics['accuracy'];
|
|
}
|
|
}
|
|
|
|
$healthIndicators['average_accuracy'] = $totalAccuracy / count($models);
|
|
|
|
if ($healthIndicators['degraded_models'] > 0) {
|
|
$healthIndicators['overall_status'] = $healthIndicators['degraded_models'] > 2 ? 'critical' : 'warning';
|
|
}
|
|
|
|
echo " → Health Indicators:\n";
|
|
echo " Overall Status: {$healthIndicators['overall_status']}\n";
|
|
echo " Healthy Models: {$healthIndicators['healthy_models']}/{" . count($models) . "}\n";
|
|
echo " Degraded Models: {$healthIndicators['degraded_models']}\n";
|
|
echo " Average Accuracy: " . sprintf("%.1f%%", $healthIndicators['average_accuracy'] * 100) . "\n";
|
|
echo " Accuracy Range: " . sprintf("%.1f%%", $healthIndicators['lowest_accuracy'] * 100) . " - " . sprintf("%.1f%%", $healthIndicators['highest_accuracy'] * 100) . "\n";
|
|
echo " Total Predictions: {$healthIndicators['total_predictions']}\n";
|
|
|
|
if (!empty($healthIndicators['models_below_threshold'])) {
|
|
echo " Models Below Threshold: " . implode(', ', $healthIndicators['models_below_threshold']) . "\n";
|
|
}
|
|
echo "\n";
|
|
|
|
// ========================================================================
|
|
// Dashboard Data 6: JSON Export for Frontend
|
|
// ========================================================================
|
|
echo "9. Generating JSON Dashboard Data...\n";
|
|
|
|
$dashboardData = [
|
|
'timestamp' => Timestamp::now()->format('Y-m-d H:i:s'),
|
|
'summary' => [
|
|
'total_models' => $registrySummary['total_models'],
|
|
'healthy_models' => $healthIndicators['healthy_models'],
|
|
'degraded_models' => $healthIndicators['degraded_models'],
|
|
'total_predictions' => $healthIndicators['total_predictions'],
|
|
'average_accuracy' => $healthIndicators['average_accuracy'],
|
|
'overall_status' => $healthIndicators['overall_status']
|
|
],
|
|
'models' => $performanceOverview,
|
|
'alerts' => $degradationAlerts,
|
|
'confusion_matrices' => $confusionMatrices,
|
|
'health' => $healthIndicators
|
|
];
|
|
|
|
$jsonData = json_encode($dashboardData, JSON_PRETTY_PRINT);
|
|
|
|
echo " ✓ JSON Dashboard Data Generated (" . strlen($jsonData) . " bytes)\n";
|
|
echo "\n";
|
|
|
|
// ========================================================================
|
|
// Display JSON Sample
|
|
// ========================================================================
|
|
echo "10. Dashboard Data Sample (JSON):\n";
|
|
echo substr($jsonData, 0, 500) . "...\n\n";
|
|
|
|
// ========================================================================
|
|
// Test Summary
|
|
// ========================================================================
|
|
echo "=== Test Summary ===\n";
|
|
echo "✓ Model Performance Overview: Collected\n";
|
|
echo "✓ Degradation Alerts: Generated\n";
|
|
echo "✓ Confusion Matrices: Calculated\n";
|
|
echo "✓ Registry Summary: Compiled\n";
|
|
echo "✓ System Health Indicators: Analyzed\n";
|
|
echo "✓ JSON Dashboard Data: Exported\n\n";
|
|
|
|
echo "Dashboard Summary:\n";
|
|
echo " - {$registrySummary['total_models']} models tracked\n";
|
|
echo " - {$healthIndicators['healthy_models']} healthy, {$healthIndicators['degraded_models']} degraded\n";
|
|
echo " - Average accuracy: " . sprintf("%.1f%%", $healthIndicators['average_accuracy'] * 100) . "\n";
|
|
echo " - {$totalPredictions} total predictions processed\n";
|
|
echo " - " . count($degradationAlerts) . " active alert(s)\n";
|
|
echo " - Overall status: {$healthIndicators['overall_status']}\n\n";
|
|
|
|
echo "=== ML Monitoring Dashboard PASSED ===\n";
|
|
|
|
} catch (\Throwable $e) {
|
|
echo "\n!!! TEST FAILED !!!\n";
|
|
echo "Error: " . $e->getMessage() . "\n";
|
|
echo "File: " . $e->getFile() . ":" . $e->getLine() . "\n";
|
|
echo "\nStack trace:\n" . $e->getTraceAsString() . "\n";
|
|
exit(1);
|
|
}
|