Michael Schiemer
3b623e7afb
- Create AnsibleDeployStage using framework's Process module for secure command execution - Integrate AnsibleDeployStage into DeploymentPipelineCommands for production deployments - Add force_deploy flag support in Ansible playbook to override stale locks - Use PHP deployment module as orchestrator (php console.php deploy:production) - Fix ErrorAggregationInitializer to use Environment class instead of $_ENV superglobal Architecture: - BuildStage → AnsibleDeployStage → HealthCheckStage for production - Process module provides timeout, error handling, and output capture - Ansible playbook supports rollback via rollback-git-based.yml - Zero-downtime deployments with health checks
456 lines
16 KiB
PHP
456 lines
16 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace App\Application\Api\MachineLearning;
|
|
|
|
use App\Framework\Attributes\Route;
|
|
use App\Framework\Core\ValueObjects\Version;
|
|
use App\Framework\Http\HttpRequest;
|
|
use App\Framework\Http\Method;
|
|
use App\Framework\Http\Status;
|
|
use App\Framework\MachineLearning\ModelManagement\ABTestingService;
|
|
use App\Framework\MachineLearning\ModelManagement\ModelRegistry;
|
|
use App\Framework\MachineLearning\ModelManagement\ValueObjects\ABTestConfig;
|
|
use App\Framework\OpenApi\Attributes\ApiEndpoint;
|
|
use App\Framework\OpenApi\Attributes\ApiParameter;
|
|
use App\Framework\OpenApi\Attributes\ApiRequestBody;
|
|
use App\Framework\OpenApi\Attributes\ApiResponse;
|
|
use App\Framework\OpenApi\Attributes\ApiSecurity;
|
|
use App\Framework\Router\Result\JsonResult;
|
|
|
|
/**
|
|
* ML A/B Testing API Controller
|
|
*
|
|
* RESTful API endpoints for A/B testing machine learning models:
|
|
* - Start A/B tests
|
|
* - Get test results
|
|
* - Generate rollout plans
|
|
* - Calculate sample sizes
|
|
*/
|
|
#[ApiSecurity('bearerAuth')]
|
|
final readonly class MLABTestingController
|
|
{
|
|
public function __construct(
|
|
private ABTestingService $abTesting,
|
|
private ModelRegistry $registry
|
|
) {}
|
|
|
|
#[Route(path: '/api/ml/ab-test', method: Method::POST)]
|
|
#[ApiEndpoint(
|
|
summary: 'Start A/B test',
|
|
description: 'Create and start an A/B test comparing two model versions',
|
|
tags: ['Machine Learning'],
|
|
)]
|
|
#[ApiRequestBody(
|
|
description: 'A/B test configuration',
|
|
required: true,
|
|
example: [
|
|
'model_name' => 'fraud-detector',
|
|
'version_a' => '1.0.0',
|
|
'version_b' => '2.0.0',
|
|
'traffic_split_a' => 0.5,
|
|
'primary_metric' => 'accuracy',
|
|
'minimum_improvement' => 0.05,
|
|
],
|
|
)]
|
|
#[ApiResponse(
|
|
statusCode: 201,
|
|
description: 'A/B test created successfully',
|
|
example: [
|
|
'test_id' => 'test_123',
|
|
'model_name' => 'fraud-detector',
|
|
'version_a' => '1.0.0',
|
|
'version_b' => '2.0.0',
|
|
'traffic_split' => [
|
|
'version_a' => 0.5,
|
|
'version_b' => 0.5,
|
|
],
|
|
'status' => 'running',
|
|
'created_at' => '2024-01-01T00:00:00Z',
|
|
],
|
|
)]
|
|
#[ApiResponse(
|
|
statusCode: 400,
|
|
description: 'Invalid test configuration',
|
|
)]
|
|
public function startTest(HttpRequest $request): JsonResult
|
|
{
|
|
try {
|
|
$data = $request->parsedBody->toArray();
|
|
|
|
// Validate required fields
|
|
if (!isset($data['model_name'], $data['version_a'], $data['version_b'])) {
|
|
return new JsonResult([
|
|
'error' => 'Missing required fields',
|
|
'required' => ['model_name', 'version_a', 'version_b'],
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
|
|
// Parse versions
|
|
$versionA = Version::fromString($data['version_a']);
|
|
$versionB = Version::fromString($data['version_b']);
|
|
|
|
// Verify models exist
|
|
$metadataA = $this->registry->get($data['model_name'], $versionA);
|
|
$metadataB = $this->registry->get($data['model_name'], $versionB);
|
|
|
|
if ($metadataA === null) {
|
|
return new JsonResult([
|
|
'error' => 'Version A not found',
|
|
'model_name' => $data['model_name'],
|
|
'version' => $data['version_a'],
|
|
], Status::NOT_FOUND);
|
|
}
|
|
|
|
if ($metadataB === null) {
|
|
return new JsonResult([
|
|
'error' => 'Version B not found',
|
|
'model_name' => $data['model_name'],
|
|
'version' => $data['version_b'],
|
|
], Status::NOT_FOUND);
|
|
}
|
|
|
|
// Create A/B test config
|
|
$config = new ABTestConfig(
|
|
modelName: $data['model_name'],
|
|
versionA: $versionA,
|
|
versionB: $versionB,
|
|
trafficSplitA: (float) ($data['traffic_split_a'] ?? 0.5),
|
|
primaryMetric: $data['primary_metric'] ?? 'accuracy',
|
|
minimumImprovement: (float) ($data['minimum_improvement'] ?? 0.05),
|
|
significanceLevel: (float) ($data['significance_level'] ?? 0.05)
|
|
);
|
|
|
|
// Generate test ID (in production, store in database)
|
|
$testId = 'test_' . bin2hex(random_bytes(8));
|
|
|
|
return new JsonResult([
|
|
'test_id' => $testId,
|
|
'model_name' => $config->modelName,
|
|
'version_a' => $config->versionA->toString(),
|
|
'version_b' => $config->versionB->toString(),
|
|
'traffic_split' => [
|
|
'version_a' => $config->trafficSplitA,
|
|
'version_b' => 1.0 - $config->trafficSplitA,
|
|
],
|
|
'primary_metric' => $config->primaryMetric,
|
|
'minimum_improvement' => $config->minimumImprovement,
|
|
'status' => 'running',
|
|
'description' => $config->getDescription(),
|
|
'created_at' => date('c'),
|
|
], Status::CREATED);
|
|
} catch (\InvalidArgumentException $e) {
|
|
return new JsonResult([
|
|
'error' => 'Invalid test configuration',
|
|
'message' => $e->getMessage(),
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
}
|
|
|
|
#[Route(path: '/api/ml/ab-test/compare', method: Method::POST)]
|
|
#[ApiEndpoint(
|
|
summary: 'Compare model versions',
|
|
description: 'Compare performance of two model versions and get winner recommendation',
|
|
tags: ['Machine Learning'],
|
|
)]
|
|
#[ApiRequestBody(
|
|
description: 'Model comparison configuration',
|
|
required: true,
|
|
example: [
|
|
'model_name' => 'fraud-detector',
|
|
'version_a' => '1.0.0',
|
|
'version_b' => '2.0.0',
|
|
'primary_metric' => 'f1_score',
|
|
'minimum_improvement' => 0.05,
|
|
],
|
|
)]
|
|
#[ApiResponse(
|
|
statusCode: 200,
|
|
description: 'Comparison completed successfully',
|
|
example: [
|
|
'winner' => 'B',
|
|
'statistically_significant' => true,
|
|
'metrics_difference' => [
|
|
'accuracy' => 0.07,
|
|
'f1_score' => 0.08,
|
|
],
|
|
'primary_metric_improvement' => 8.5,
|
|
'recommendation' => 'Version B wins with 8.5% improvement - deploy new version',
|
|
'summary' => 'Version B significantly outperforms Version A',
|
|
],
|
|
)]
|
|
#[ApiResponse(
|
|
statusCode: 404,
|
|
description: 'Model version not found',
|
|
)]
|
|
public function compareVersions(HttpRequest $request): JsonResult
|
|
{
|
|
try {
|
|
$data = $request->parsedBody->toArray();
|
|
|
|
if (!isset($data['model_name'], $data['version_a'], $data['version_b'])) {
|
|
return new JsonResult([
|
|
'error' => 'Missing required fields',
|
|
'required' => ['model_name', 'version_a', 'version_b'],
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
|
|
$versionA = Version::fromString($data['version_a']);
|
|
$versionB = Version::fromString($data['version_b']);
|
|
|
|
$config = new ABTestConfig(
|
|
modelName: $data['model_name'],
|
|
versionA: $versionA,
|
|
versionB: $versionB,
|
|
trafficSplitA: 0.5,
|
|
primaryMetric: $data['primary_metric'] ?? 'accuracy',
|
|
minimumImprovement: (float) ($data['minimum_improvement'] ?? 0.05)
|
|
);
|
|
|
|
// Run comparison
|
|
$result = $this->abTesting->runTest($config);
|
|
|
|
return new JsonResult([
|
|
'winner' => $result->winner,
|
|
'statistically_significant' => $result->isStatisticallySignificant,
|
|
'metrics_difference' => $result->metricsDifference,
|
|
'primary_metric_improvement' => $result->getPrimaryMetricImprovementPercent(),
|
|
'recommendation' => $result->recommendation,
|
|
'summary' => $result->getSummary(),
|
|
'should_deploy_version_b' => $result->shouldDeployVersionB(),
|
|
'is_inconclusive' => $result->isInconclusive(),
|
|
]);
|
|
} catch (\InvalidArgumentException $e) {
|
|
return new JsonResult([
|
|
'error' => 'Invalid comparison parameters',
|
|
'message' => $e->getMessage(),
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
}
|
|
|
|
#[Route(path: '/api/ml/ab-test/rollout-plan', method: Method::POST)]
|
|
#[ApiEndpoint(
|
|
summary: 'Generate rollout plan',
|
|
description: 'Generate a gradual rollout plan for deploying a new model version',
|
|
tags: ['Machine Learning'],
|
|
)]
|
|
#[ApiRequestBody(
|
|
description: 'Rollout configuration',
|
|
required: true,
|
|
example: [
|
|
'model_name' => 'fraud-detector',
|
|
'current_version' => '1.0.0',
|
|
'new_version' => '2.0.0',
|
|
'steps' => 5,
|
|
],
|
|
)]
|
|
#[ApiResponse(
|
|
statusCode: 200,
|
|
description: 'Rollout plan generated successfully',
|
|
example: [
|
|
'model_name' => 'fraud-detector',
|
|
'current_version' => '1.0.0',
|
|
'new_version' => '2.0.0',
|
|
'rollout_stages' => [
|
|
[
|
|
'stage' => 1,
|
|
'current_version_traffic' => 80,
|
|
'new_version_traffic' => 20,
|
|
],
|
|
[
|
|
'stage' => 2,
|
|
'current_version_traffic' => 60,
|
|
'new_version_traffic' => 40,
|
|
],
|
|
],
|
|
],
|
|
)]
|
|
public function generateRolloutPlan(HttpRequest $request): JsonResult
|
|
{
|
|
try {
|
|
$data = $request->parsedBody->toArray();
|
|
|
|
if (!isset($data['model_name'], $data['current_version'], $data['new_version'])) {
|
|
return new JsonResult([
|
|
'error' => 'Missing required fields',
|
|
'required' => ['model_name', 'current_version', 'new_version'],
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
|
|
$steps = (int) ($data['steps'] ?? 5);
|
|
|
|
if ($steps < 2 || $steps > 10) {
|
|
return new JsonResult([
|
|
'error' => 'Steps must be between 2 and 10',
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
|
|
// Generate rollout plan
|
|
$plan = $this->abTesting->generateRolloutPlan($steps);
|
|
|
|
// Format response
|
|
$stages = [];
|
|
foreach ($plan as $stage => $newVersionTraffic) {
|
|
$stages[] = [
|
|
'stage' => $stage,
|
|
'current_version_traffic' => (int) ((1.0 - $newVersionTraffic) * 100),
|
|
'new_version_traffic' => (int) ($newVersionTraffic * 100),
|
|
];
|
|
}
|
|
|
|
return new JsonResult([
|
|
'model_name' => $data['model_name'],
|
|
'current_version' => $data['current_version'],
|
|
'new_version' => $data['new_version'],
|
|
'total_stages' => $steps,
|
|
'rollout_stages' => $stages,
|
|
'recommendation' => 'Monitor performance at each stage before proceeding to next',
|
|
]);
|
|
} catch (\InvalidArgumentException $e) {
|
|
return new JsonResult([
|
|
'error' => 'Invalid rollout configuration',
|
|
'message' => $e->getMessage(),
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
}
|
|
|
|
#[Route(path: '/api/ml/ab-test/sample-size', method: Method::GET)]
|
|
#[ApiEndpoint(
|
|
summary: 'Calculate required sample size',
|
|
description: 'Calculate the required sample size for statistically significant A/B test',
|
|
tags: ['Machine Learning'],
|
|
)]
|
|
#[ApiParameter(
|
|
name: 'confidence_level',
|
|
in: 'query',
|
|
description: 'Confidence level (0.90, 0.95, 0.99)',
|
|
required: false,
|
|
type: 'number',
|
|
example: 0.95,
|
|
)]
|
|
#[ApiParameter(
|
|
name: 'margin_of_error',
|
|
in: 'query',
|
|
description: 'Margin of error (typically 0.01-0.10)',
|
|
required: false,
|
|
type: 'number',
|
|
example: 0.05,
|
|
)]
|
|
#[ApiResponse(
|
|
statusCode: 200,
|
|
description: 'Sample size calculated successfully',
|
|
example: [
|
|
'required_samples_per_version' => 385,
|
|
'total_samples_needed' => 770,
|
|
'confidence_level' => 0.95,
|
|
'margin_of_error' => 0.05,
|
|
'recommendation' => 'Collect at least 385 predictions per version',
|
|
],
|
|
)]
|
|
public function calculateSampleSize(HttpRequest $request): JsonResult
|
|
{
|
|
$confidenceLevel = (float) ($request->queryParameters['confidence_level'] ?? 0.95);
|
|
$marginOfError = (float) ($request->queryParameters['margin_of_error'] ?? 0.05);
|
|
|
|
// Validate parameters
|
|
if ($confidenceLevel < 0.5 || $confidenceLevel > 0.99) {
|
|
return new JsonResult([
|
|
'error' => 'Confidence level must be between 0.5 and 0.99',
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
|
|
if ($marginOfError < 0.01 || $marginOfError > 0.20) {
|
|
return new JsonResult([
|
|
'error' => 'Margin of error must be between 0.01 and 0.20',
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
|
|
// Calculate sample size
|
|
$samplesPerVersion = $this->abTesting->calculateRequiredSampleSize(
|
|
$confidenceLevel,
|
|
$marginOfError
|
|
);
|
|
|
|
return new JsonResult([
|
|
'required_samples_per_version' => $samplesPerVersion,
|
|
'total_samples_needed' => $samplesPerVersion * 2,
|
|
'confidence_level' => $confidenceLevel,
|
|
'margin_of_error' => $marginOfError,
|
|
'confidence_level_percent' => ($confidenceLevel * 100) . '%',
|
|
'margin_of_error_percent' => ($marginOfError * 100) . '%',
|
|
'recommendation' => "Collect at least {$samplesPerVersion} predictions per version for statistically significant results",
|
|
]);
|
|
}
|
|
|
|
#[Route(path: '/api/ml/ab-test/select-version', method: Method::POST)]
|
|
#[ApiEndpoint(
|
|
summary: 'Select model version for traffic routing',
|
|
description: 'Randomly select a model version based on A/B test traffic split configuration',
|
|
tags: ['Machine Learning'],
|
|
)]
|
|
#[ApiRequestBody(
|
|
description: 'Traffic routing configuration',
|
|
required: true,
|
|
example: [
|
|
'model_name' => 'fraud-detector',
|
|
'version_a' => '1.0.0',
|
|
'version_b' => '2.0.0',
|
|
'traffic_split_a' => 0.8,
|
|
],
|
|
)]
|
|
#[ApiResponse(
|
|
statusCode: 200,
|
|
description: 'Version selected successfully',
|
|
example: [
|
|
'selected_version' => '2.0.0',
|
|
'model_name' => 'fraud-detector',
|
|
'traffic_split' => [
|
|
'version_a' => 0.8,
|
|
'version_b' => 0.2,
|
|
],
|
|
],
|
|
)]
|
|
public function selectVersion(HttpRequest $request): JsonResult
|
|
{
|
|
try {
|
|
$data = $request->parsedBody->toArray();
|
|
|
|
if (!isset($data['model_name'], $data['version_a'], $data['version_b'])) {
|
|
return new JsonResult([
|
|
'error' => 'Missing required fields',
|
|
'required' => ['model_name', 'version_a', 'version_b'],
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
|
|
$versionA = Version::fromString($data['version_a']);
|
|
$versionB = Version::fromString($data['version_b']);
|
|
|
|
$config = new ABTestConfig(
|
|
modelName: $data['model_name'],
|
|
versionA: $versionA,
|
|
versionB: $versionB,
|
|
trafficSplitA: (float) ($data['traffic_split_a'] ?? 0.5),
|
|
primaryMetric: 'accuracy'
|
|
);
|
|
|
|
// Select version based on traffic split
|
|
$selectedVersion = $this->abTesting->selectVersion($config);
|
|
|
|
return new JsonResult([
|
|
'selected_version' => $selectedVersion->toString(),
|
|
'model_name' => $config->modelName,
|
|
'traffic_split' => [
|
|
'version_a' => $config->trafficSplitA,
|
|
'version_b' => 1.0 - $config->trafficSplitA,
|
|
],
|
|
]);
|
|
} catch (\InvalidArgumentException $e) {
|
|
return new JsonResult([
|
|
'error' => 'Invalid routing configuration',
|
|
'message' => $e->getMessage(),
|
|
], Status::BAD_REQUEST);
|
|
}
|
|
}
|
|
}
|