tokenizer = new PhpTokenizer(); } /** * Run all test cases */ public function runAll(): void { $testCases = $this->getTestCases(); $total = count($testCases); $passed = 0; $failed = 0; echo "=== Token Classification Test Suite ===\n\n"; echo "Running {$total} test cases...\n\n"; foreach ($testCases as $index => $testCase) { echo str_repeat('=', 80) . "\n"; echo "Test Case " . ($index + 1) . ": {$testCase['name']}\n"; echo str_repeat('=', 80) . "\n\n"; $result = $this->runTestCase($testCase); if ($result['passed']) { $passed++; echo "✅ PASSED\n\n"; } else { $failed++; echo "❌ FAILED\n\n"; } } echo str_repeat('=', 80) . "\n"; echo "Summary: {$passed}/{$total} passed, {$failed}/{$total} failed\n"; } /** * Run a single test case */ private function runTestCase(array $testCase): array { $code = $testCase['code']; $expected = $testCase['expected']; $expectedMetadata = $testCase['expectedMetadata'] ?? null; echo "Code:\n"; $codeLines = explode("\n", $code); foreach ($codeLines as $i => $line) { echo sprintf(" %2d: %s\n", $i + 1, $line); } echo "\n"; // Tokenize the code $tokens = $this->tokenizer->tokenize($code); // Build actual token map $actual = $this->buildTokenMap($tokens); // Compare expected vs actual $differences = $this->compareTokens($expected, $actual); // Check metadata if expected $metadataDifferences = []; if ($expectedMetadata !== null) { $metadataDifferences = $this->compareMetadata($tokens, $expectedMetadata); } // Display expected tokens echo "Expected Tokens:\n"; foreach ($expected as $lineNum => $lineTokens) { foreach ($lineTokens as $tokenValue => $expectedType) { echo sprintf(" Line %d: '%s' → %s\n", $lineNum, $tokenValue, $expectedType); } } echo "\n"; // Display actual tokens echo "Actual Tokens:\n"; foreach ($actual as $lineNum => $lineTokens) { foreach ($lineTokens as $tokenValue => $actualType) { echo sprintf(" Line %d: '%s' → %s\n", $lineNum, $tokenValue, $actualType->value); } } echo "\n"; // Display differences if (empty($differences) && empty($metadataDifferences)) { echo "Differences: None - All tokens and metadata match!\n"; return ['passed' => true, 'differences' => []]; } echo "Differences:\n"; foreach ($differences as $diff) { if ($diff['match']) { echo sprintf(" ✅ Line %d: '%s' → %s (correct)\n", $diff['line'], $diff['token'], $diff['expected'] ); } else { echo sprintf(" ❌ Line %d: '%s' → Expected %s, got %s\n", $diff['line'], $diff['token'], $diff['expected'], $diff['actual'] ); } } // Display metadata differences if (!empty($metadataDifferences)) { echo "\nMetadata Differences:\n"; foreach ($metadataDifferences as $diff) { if ($diff['match']) { echo sprintf(" ✅ Line %d: '%s' → %s (correct)\n", $diff['line'], $diff['token'], $diff['expected'] ); } else { echo sprintf(" ❌ Line %d: '%s' → Expected %s, got %s\n", $diff['line'], $diff['token'], $diff['expected'], $diff['actual'] ); } } } $allDifferences = array_merge($differences, $metadataDifferences); return [ 'passed' => empty(array_filter($allDifferences, fn($d) => !$d['match'])), 'differences' => $allDifferences ]; } /** * Build token map from TokenCollection */ private function buildTokenMap($tokens): array { $map = []; foreach ($tokens as $token) { $line = $token->line; $value = trim($token->value); // Skip whitespace-only tokens and PHP tags if ($token->type === TokenType::WHITESPACE || $token->type === TokenType::PHP_TAG || empty($value)) { continue; } if (!isset($map[$line])) { $map[$line] = []; } // Use token value as key, token type as value $map[$line][$value] = $token->type; } return $map; } /** * Compare metadata for tokens */ private function compareMetadata($tokens, array $expectedMetadata): array { $differences = []; // Build token lookup by line and value $tokenLookup = []; foreach ($tokens as $token) { $line = $token->line; $value = trim($token->value); if (empty($value) || $token->type === TokenType::WHITESPACE || $token->type === TokenType::PHP_TAG) { continue; } if (!isset($tokenLookup[$line])) { $tokenLookup[$line] = []; } $tokenLookup[$line][$value] = $token; } // Compare expected metadata foreach ($expectedMetadata as $lineNum => $lineMetadata) { foreach ($lineMetadata as $tokenValue => $expectedMeta) { $token = $tokenLookup[$lineNum][$tokenValue] ?? null; if ($token === null) { $differences[] = [ 'line' => $lineNum, 'token' => $tokenValue, 'expected' => json_encode($expectedMeta), 'actual' => 'TOKEN_NOT_FOUND', 'match' => false ]; continue; } $actualMeta = $token->metadata; // Check each expected metadata property foreach ($expectedMeta as $property => $expectedValue) { $actualValue = $actualMeta?->$property ?? null; if ($property === 'isBuiltIn') { $actualValue = $actualMeta?->isBuiltIn ?? false; } $match = $actualValue === $expectedValue; if (!$match) { $differences[] = [ 'line' => $lineNum, 'token' => $tokenValue, 'expected' => "{$property}: " . ($expectedValue === true ? 'true' : ($expectedValue === false ? 'false' : $expectedValue)), 'actual' => "{$property}: " . ($actualValue === true ? 'true' : ($actualValue === false ? 'false' : ($actualValue ?? 'null'))), 'match' => false ]; } } } } return $differences; } /** * Compare expected and actual tokens */ private function compareTokens(array $expected, array $actual): array { $differences = []; $allLines = array_unique(array_merge(array_keys($expected), array_keys($actual))); foreach ($allLines as $lineNum) { $expectedTokens = $expected[$lineNum] ?? []; $actualTokens = $actual[$lineNum] ?? []; // Check all expected tokens foreach ($expectedTokens as $tokenValue => $expectedTypeName) { $expectedType = TokenType::tryFrom($expectedTypeName); $actualType = $actualTokens[$tokenValue] ?? null; $differences[] = [ 'line' => $lineNum, 'token' => $tokenValue, 'expected' => $expectedTypeName, 'actual' => $actualType?->value ?? 'NOT_FOUND', 'match' => $actualType === $expectedType ]; } // Check for unexpected tokens (optional - could be verbose) foreach ($actualTokens as $tokenValue => $actualType) { if (!isset($expectedTokens[$tokenValue])) { // This token wasn't expected - could log as info } } } return $differences; } /** * Get all test cases */ private function getTestCases(): array { return array_merge( // Test Cases for Classes $this->getClassTestCases(), // Test Cases for Enums $this->getEnumTestCases(), // Test Cases for Attributes $this->getAttributeTestCases(), // Test Cases for Methods and Properties $this->getMethodPropertyTestCases(), // Test Cases for Metadata $this->getMetadataTestCases() ); } /** * Get test cases for classes */ private function getClassTestCases(): array { return [ [ 'name' => 'Simple Class', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Final Class', 'code' => ' [ 1 => [ 'final' => 'keyword_modifier', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Abstract Class', 'code' => ' [ 1 => [ 'abstract' => 'keyword_modifier', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Readonly Class', 'code' => ' [ 1 => [ 'readonly' => 'keyword_modifier', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Class with Extends', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'Child' => 'class_name', 'extends' => 'keyword_other', 'Parent' => 'class_name', ], ], ], [ 'name' => 'Class with Implements', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'MyClass' => 'class_name', 'implements' => 'keyword_other', 'MyInterface' => 'interface_name', ], ], ], [ 'name' => 'Class with Multiple Modifiers', 'code' => ' [ 1 => [ 'final' => 'keyword_modifier', 'readonly' => 'keyword_modifier', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Class with Extends and Implements', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'Child' => 'class_name', 'extends' => 'keyword_other', 'Parent' => 'class_name', 'implements' => 'keyword_other', 'MyInterface' => 'interface_name', ], ], ], ]; } /** * Get test cases for enums */ private function getEnumTestCases(): array { return [ [ 'name' => 'Pure Enum', 'code' => ' [ 1 => [ 'enum' => 'keyword_other', 'Status' => 'enum_name', ], ], ], [ 'name' => 'Backed Enum (String)', 'code' => ' [ 1 => [ 'enum' => 'keyword_other', 'Status' => 'enum_name', ':' => 'operator', 'string' => 'keyword_type', ], ], ], [ 'name' => 'Backed Enum (Int)', 'code' => ' [ 1 => [ 'enum' => 'keyword_other', 'Status' => 'enum_name', ':' => 'operator', 'int' => 'keyword_type', ], ], ], [ 'name' => 'Enum with Implements', 'code' => ' [ 1 => [ 'enum' => 'keyword_other', 'Status' => 'enum_name', 'implements' => 'keyword_other', 'MyInterface' => 'interface_name', ], ], ], [ 'name' => 'Final Enum', 'code' => ' [ 1 => [ 'final' => 'keyword_modifier', 'enum' => 'keyword_other', 'Status' => 'enum_name', ], ], ], [ 'name' => 'Backed Enum with Implements', 'code' => ' [ 1 => [ 'enum' => 'keyword_other', 'Status' => 'enum_name', ':' => 'operator', 'string' => 'keyword_type', 'implements' => 'keyword_other', 'MyInterface' => 'interface_name', ], ], ], ]; } /** * Get test cases for attributes */ private function getAttributeTestCases(): array { return [ [ 'name' => 'Simple Attribute', 'code' => ' [ 1 => [ '#' => 'attribute', '[' => 'bracket', 'Route' => 'attribute_name', ']' => 'bracket', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Attribute with Parameter', 'code' => ' [ 1 => [ '#' => 'attribute', '[' => 'bracket', 'Route' => 'attribute_name', '(' => 'parenthesis', '\'/api\'' => 'string_literal', ')' => 'parenthesis', ']' => 'bracket', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Attribute with Named Parameters', 'code' => ' [ 1 => [ '#' => 'attribute', '[' => 'bracket', 'Route' => 'attribute_name', '(' => 'parenthesis', 'path' => 'attribute_name', ':' => 'operator', '\'/api\'' => 'string_literal', ',' => 'punctuation', 'method' => 'attribute_name', ':' => 'operator', '\'GET\'' => 'string_literal', ')' => 'parenthesis', ']' => 'bracket', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Multiple Attributes', 'code' => ' [ 1 => [ '#' => 'attribute', '[' => 'bracket', 'Route' => 'attribute_name', ',' => 'punctuation', 'Auth' => 'attribute_name', ']' => 'bracket', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Attribute with Multiple Parameters', 'code' => ' [ 1 => [ '#' => 'attribute', '[' => 'bracket', 'Route' => 'attribute_name', '(' => 'parenthesis', '\'/api\'' => 'string_literal', ',' => 'punctuation', '\'POST\'' => 'string_literal', ')' => 'parenthesis', ']' => 'bracket', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Attribute with Class Parameter', 'code' => ' [ 1 => [ '#' => 'attribute', '[' => 'bracket', 'Route' => 'attribute_name', '(' => 'parenthesis', 'MyClass' => 'class_name', '::' => 'operator', 'class' => 'keyword_other', ')' => 'parenthesis', ']' => 'bracket', 'class' => 'keyword_other', 'MyClass' => 'class_name', ], ], ], [ 'name' => 'Attribute on Method', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'MyClass' => 'class_name', '{' => 'brace', ], 2 => [ '#' => 'attribute', '[' => 'bracket', 'Route' => 'attribute_name', ']' => 'bracket', 'public' => 'keyword_modifier', 'function' => 'keyword_other', 'test' => 'method_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], ]; } /** * Get test cases for methods and properties */ private function getMethodPropertyTestCases(): array { return [ [ 'name' => 'Static Method Call', 'code' => ' [ 1 => [ 'MyClass' => 'class_name', '::' => 'operator', 'staticMethod' => 'static_method_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], [ 'name' => 'Instance Method Call', 'code' => 'instanceMethod();', 'expected' => [ 1 => [ '$obj' => 'variable', '->' => 'operator', 'instanceMethod' => 'instance_method_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], [ 'name' => 'Property Access', 'code' => 'property;', 'expected' => [ 1 => [ '$obj' => 'variable', '->' => 'operator', 'property' => 'instance_property_name', ], ], ], [ 'name' => 'Static Property Access', 'code' => ' [ 1 => [ 'MyClass' => 'class_name', '::' => 'operator', '$staticProperty' => 'variable', ], ], ], [ 'name' => 'Constructor Call', 'code' => ' [ 1 => [ 'new' => 'keyword_other', 'MyClass' => 'constructor_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], [ 'name' => 'Method Definition', 'code' => ' [ 1 => [ 'public' => 'keyword_modifier', 'function' => 'keyword_other', 'myMethod' => 'function_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], [ 'name' => 'Method Definition in Class', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'MyClass' => 'class_name', '{' => 'brace', ], 2 => [ 'public' => 'keyword_modifier', 'function' => 'keyword_other', 'myMethod' => 'method_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], [ 'name' => 'Constructor Definition', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'MyClass' => 'class_name', '{' => 'brace', ], 2 => [ 'public' => 'keyword_modifier', 'function' => 'keyword_other', '__construct' => 'constructor_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], [ 'name' => 'Static Method Definition', 'code' => ' [ 1 => [ 'class' => 'keyword_other', 'MyClass' => 'class_name', '{' => 'brace', ], 2 => [ 'public' => 'keyword_modifier', 'static' => 'keyword_modifier', 'function' => 'keyword_other', 'staticMethod' => 'method_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], [ 'name' => 'Nullsafe Operator Method Call', 'code' => 'method();', 'expected' => [ 1 => [ '$obj' => 'variable', '?->' => 'operator', 'method' => 'instance_method_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], ], ]; } /** * Get test cases for metadata */ private function getMetadataTestCases(): array { return [ [ 'name' => 'Built-in Function Call', 'code' => ' [ 1 => [ 'count' => 'function_name', '$array' => 'variable', ], ], 'expectedMetadata' => [ 1 => [ 'count' => [ 'functionName' => 'count', 'isBuiltIn' => true, ], ], ], ], [ 'name' => 'Built-in Function strlen', 'code' => ' [ 1 => [ 'strlen' => 'function_name', '$str' => 'variable', ], ], 'expectedMetadata' => [ 1 => [ 'strlen' => [ 'functionName' => 'strlen', 'isBuiltIn' => true, ], ], ], ], [ 'name' => 'Built-in Function array_map', 'code' => ' [ 1 => [ 'array_map' => 'function_name', '$callback' => 'variable', '$array' => 'variable', ], ], 'expectedMetadata' => [ 1 => [ 'array_map' => [ 'functionName' => 'array_map', 'isBuiltIn' => true, ], ], ], ], [ 'name' => 'User Function Call', 'code' => ' [ 1 => [ 'myFunction' => 'function_name', '(' => 'parenthesis', ')' => 'parenthesis', ], ], 'expectedMetadata' => [ 1 => [ 'myFunction' => [ 'functionName' => 'myFunction', 'isBuiltIn' => false, ], ], ], ], [ 'name' => 'Static Method Call with Metadata', 'code' => ' [ 1 => [ 'MyClass' => 'class_name', '::' => 'operator', 'staticMethod' => 'static_method_name', ], ], 'expectedMetadata' => [ 1 => [ 'MyClass' => [ 'className' => 'MyClass', ], 'staticMethod' => [ 'functionName' => 'staticMethod', 'className' => 'MyClass', ], ], ], ], [ 'name' => 'Instance Method Call with Metadata', 'code' => 'instanceMethod();', 'expected' => [ 1 => [ '$obj' => 'variable', '->' => 'operator', 'instanceMethod' => 'instance_method_name', ], ], 'expectedMetadata' => [ 1 => [ 'instanceMethod' => [ 'functionName' => 'instanceMethod', ], ], ], ], ]; } } // Run tests if executed directly if (php_sapi_name() === 'cli') { $test = new TokenClassificationTest(); $test->runAll(); }