Michael Schiemer
fc3d7e6357
- Add comprehensive health check system with multiple endpoints - Add Prometheus metrics endpoint - Add production logging configurations (5 strategies) - Add complete deployment documentation suite: * QUICKSTART.md - 30-minute deployment guide * DEPLOYMENT_CHECKLIST.md - Printable verification checklist * DEPLOYMENT_WORKFLOW.md - Complete deployment lifecycle * PRODUCTION_DEPLOYMENT.md - Comprehensive technical reference * production-logging.md - Logging configuration guide * ANSIBLE_DEPLOYMENT.md - Infrastructure as Code automation * README.md - Navigation hub * DEPLOYMENT_SUMMARY.md - Executive summary - Add deployment scripts and automation - Add DEPLOYMENT_PLAN.md - Concrete plan for immediate deployment - Update README with production-ready features All production infrastructure is now complete and ready for deployment.
836 lines
32 KiB
PHP
836 lines
32 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
use App\Framework\Core\ValueObjects\Duration;
|
|
use App\Framework\Queue\ValueObjects\JobId;
|
|
use App\Framework\Queue\ValueObjects\LockKey;
|
|
use App\Framework\Queue\ValueObjects\QueueName;
|
|
use App\Framework\Queue\ValueObjects\WorkerId;
|
|
|
|
describe('LockKey Value Object', function () {
|
|
|
|
describe('Construction and Validation', function () {
|
|
it('can create valid lock keys', function () {
|
|
$validKeys = [
|
|
'simple-lock',
|
|
'job.12345',
|
|
'queue.email_processing',
|
|
'worker.worker-123',
|
|
'resource.database_connection',
|
|
'batch.monthly_report_2024',
|
|
'a', // minimum length
|
|
str_repeat('a', 255), // maximum length
|
|
];
|
|
|
|
foreach ($validKeys as $key) {
|
|
$lockKey = LockKey::fromString($key);
|
|
expect($lockKey->toString())->toBe($key);
|
|
}
|
|
});
|
|
|
|
it('rejects invalid lock keys', function () {
|
|
$invalidKeys = [
|
|
'', // empty
|
|
str_repeat('a', 256), // too long
|
|
'lock with spaces',
|
|
'lock@invalid',
|
|
'lock#invalid',
|
|
'lock$invalid',
|
|
'lock%invalid',
|
|
'lock&invalid',
|
|
'lock*invalid',
|
|
'lock(invalid)',
|
|
'lock+invalid',
|
|
'lock=invalid',
|
|
'lock[invalid]',
|
|
'lock{invalid}',
|
|
'lock|invalid',
|
|
'lock\\invalid',
|
|
'lock:invalid',
|
|
'lock;invalid',
|
|
'lock"invalid',
|
|
'lock\'invalid',
|
|
'lock<invalid>',
|
|
'lock,invalid',
|
|
'lock?invalid',
|
|
'lock/invalid',
|
|
'lock~invalid',
|
|
'lock`invalid',
|
|
];
|
|
|
|
foreach ($invalidKeys as $key) {
|
|
expect(fn () => LockKey::fromString($key))
|
|
->toThrow(\InvalidArgumentException::class);
|
|
}
|
|
});
|
|
|
|
it('allows valid characters', function () {
|
|
$validChars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.';
|
|
$lockKey = LockKey::fromString($validChars);
|
|
|
|
expect($lockKey->toString())->toBe($validChars);
|
|
});
|
|
|
|
it('is readonly and immutable', function () {
|
|
$lockKey = LockKey::fromString('test-lock');
|
|
|
|
$reflection = new ReflectionClass($lockKey);
|
|
expect($reflection->isReadOnly())->toBeTrue();
|
|
|
|
$valueProperty = $reflection->getProperty('value');
|
|
expect($valueProperty->isReadOnly())->toBeTrue();
|
|
});
|
|
});
|
|
|
|
describe('Factory Methods', function () {
|
|
it('creates lock key for job', function () {
|
|
$jobId = JobId::fromString('job-12345');
|
|
$lockKey = LockKey::forJob($jobId);
|
|
|
|
expect($lockKey->toString())->toBe('job.job-12345');
|
|
});
|
|
|
|
it('creates lock key for queue', function () {
|
|
$queueName = QueueName::fromString('email-processing');
|
|
$lockKey = LockKey::forQueue($queueName);
|
|
|
|
expect($lockKey->toString())->toBe('queue.email-processing');
|
|
});
|
|
|
|
it('creates lock key for worker', function () {
|
|
$workerId = WorkerId::fromString('worker-123');
|
|
$lockKey = LockKey::forWorker($workerId);
|
|
|
|
expect($lockKey->toString())->toBe('worker.worker-123');
|
|
});
|
|
|
|
it('creates lock key for resource', function () {
|
|
$lockKey = LockKey::forResource('database', 'primary_db');
|
|
|
|
expect($lockKey->toString())->toBe('database.primary_db');
|
|
});
|
|
|
|
it('creates lock key for batch', function () {
|
|
$lockKey = LockKey::forBatch('monthly-report-2024-01');
|
|
|
|
expect($lockKey->toString())->toBe('batch.monthly-report-2024-01');
|
|
});
|
|
});
|
|
|
|
describe('String Operations', function () {
|
|
it('equals() compares lock keys correctly', function () {
|
|
$key1 = LockKey::fromString('test-lock');
|
|
$key2 = LockKey::fromString('test-lock');
|
|
$key3 = LockKey::fromString('other-lock');
|
|
|
|
expect($key1->equals($key2))->toBeTrue();
|
|
expect($key1->equals($key3))->toBeFalse();
|
|
expect($key2->equals($key3))->toBeFalse();
|
|
});
|
|
|
|
it('string representation works correctly', function () {
|
|
$value = 'test-lock-key';
|
|
$lockKey = LockKey::fromString($value);
|
|
|
|
expect($lockKey->toString())->toBe($value);
|
|
expect((string) $lockKey)->toBe($value);
|
|
expect($lockKey->jsonSerialize())->toBe($value);
|
|
});
|
|
|
|
it('withPrefix() creates new instance with prefix', function () {
|
|
$original = LockKey::fromString('test-lock');
|
|
$prefixed = $original->withPrefix('system');
|
|
|
|
expect($prefixed->toString())->toBe('system.test-lock');
|
|
expect($original->toString())->toBe('test-lock'); // Original unchanged
|
|
});
|
|
|
|
it('withSuffix() creates new instance with suffix', function () {
|
|
$original = LockKey::fromString('test-lock');
|
|
$suffixed = $original->withSuffix('v2');
|
|
|
|
expect($suffixed->toString())->toBe('test-lock.v2');
|
|
expect($original->toString())->toBe('test-lock'); // Original unchanged
|
|
});
|
|
|
|
it('can chain prefix and suffix', function () {
|
|
$original = LockKey::fromString('lock');
|
|
$chained = $original->withPrefix('system')->withSuffix('temp');
|
|
|
|
expect($chained->toString())->toBe('system.lock.temp');
|
|
expect($original->toString())->toBe('lock'); // Original unchanged
|
|
});
|
|
|
|
it('matches() supports pattern matching', function () {
|
|
$lockKey = LockKey::fromString('job.email.batch-123');
|
|
|
|
expect($lockKey->matches('job.*'))->toBeTrue();
|
|
expect($lockKey->matches('job.email.*'))->toBeTrue();
|
|
expect($lockKey->matches('*.batch-123'))->toBeTrue();
|
|
expect($lockKey->matches('*email*'))->toBeTrue();
|
|
expect($lockKey->matches('worker.*'))->toBeFalse();
|
|
expect($lockKey->matches('*.queue.*'))->toBeFalse();
|
|
});
|
|
});
|
|
|
|
describe('Edge Cases', function () {
|
|
it('handles minimum and maximum length keys', function () {
|
|
$minKey = LockKey::fromString('x');
|
|
expect($minKey->toString())->toBe('x');
|
|
|
|
$maxKey = LockKey::fromString(str_repeat('a', 255));
|
|
expect($maxKey->toString())->toBe(str_repeat('a', 255));
|
|
});
|
|
|
|
it('validates prefix and suffix additions', function () {
|
|
$baseKey = LockKey::fromString(str_repeat('a', 250)); // Near max length
|
|
|
|
// This should work (255 chars total)
|
|
$withShortPrefix = $baseKey->withPrefix('x');
|
|
expect(strlen($withShortPrefix->toString()))->toBe(252); // 'x' + '.' + 250
|
|
|
|
// This should fail (would exceed 255 chars)
|
|
expect(fn () => $baseKey->withPrefix('toolong'))
|
|
->toThrow(\InvalidArgumentException::class);
|
|
});
|
|
|
|
it('handles complex key structures', function () {
|
|
$complexKey = LockKey::fromString('system.queue.email-processing.worker-123.batch-456');
|
|
|
|
expect($complexKey->matches('system.*'))->toBeTrue();
|
|
expect($complexKey->matches('*.queue.*'))->toBeTrue();
|
|
expect($complexKey->matches('*email-processing*'))->toBeTrue();
|
|
expect($complexKey->matches('*batch-456'))->toBeTrue();
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Distributed Lock Mock Implementation', function () {
|
|
|
|
beforeEach(function () {
|
|
// Create a mock distributed lock for testing
|
|
$this->distributedLock = new class () {
|
|
private array $locks = [];
|
|
|
|
public function acquire(LockKey $key, WorkerId $workerId, Duration $ttl): bool
|
|
{
|
|
$keyStr = $key->toString();
|
|
$now = time();
|
|
|
|
// Check if lock already exists and is not expired
|
|
if (isset($this->locks[$keyStr])) {
|
|
$lock = $this->locks[$keyStr];
|
|
if ($lock['expires_at'] > $now) {
|
|
return false; // Lock already held
|
|
}
|
|
}
|
|
|
|
// Acquire lock
|
|
$this->locks[$keyStr] = [
|
|
'worker_id' => $workerId->toString(),
|
|
'acquired_at' => $now,
|
|
'expires_at' => $now + $ttl->toSeconds(),
|
|
'ttl' => $ttl->toSeconds(),
|
|
];
|
|
|
|
return true;
|
|
}
|
|
|
|
public function extend(LockKey $key, WorkerId $workerId, Duration $ttl): bool
|
|
{
|
|
$keyStr = $key->toString();
|
|
$now = time();
|
|
|
|
if (! isset($this->locks[$keyStr])) {
|
|
return false; // Lock doesn't exist
|
|
}
|
|
|
|
$lock = $this->locks[$keyStr];
|
|
|
|
// Only the lock owner can extend it and it must not be expired
|
|
if ($lock['worker_id'] !== $workerId->toString() || $lock['expires_at'] <= $now) {
|
|
return false;
|
|
}
|
|
|
|
// Extend the lock
|
|
$this->locks[$keyStr]['expires_at'] = $now + $ttl->toSeconds();
|
|
$this->locks[$keyStr]['ttl'] = $ttl->toSeconds();
|
|
|
|
return true;
|
|
}
|
|
|
|
public function release(LockKey $key, WorkerId $workerId): bool
|
|
{
|
|
$keyStr = $key->toString();
|
|
|
|
if (! isset($this->locks[$keyStr])) {
|
|
return false; // Lock doesn't exist
|
|
}
|
|
|
|
$lock = $this->locks[$keyStr];
|
|
|
|
// Only the lock owner can release it
|
|
if ($lock['worker_id'] !== $workerId->toString()) {
|
|
return false;
|
|
}
|
|
|
|
unset($this->locks[$keyStr]);
|
|
|
|
return true;
|
|
}
|
|
|
|
public function exists(LockKey $key): bool
|
|
{
|
|
$keyStr = $key->toString();
|
|
$now = time();
|
|
|
|
if (! isset($this->locks[$keyStr])) {
|
|
return false;
|
|
}
|
|
|
|
$lock = $this->locks[$keyStr];
|
|
|
|
return $lock['expires_at'] > $now;
|
|
}
|
|
|
|
public function getLockInfo(LockKey $key): ?array
|
|
{
|
|
$keyStr = $key->toString();
|
|
$now = time();
|
|
|
|
if (! isset($this->locks[$keyStr])) {
|
|
return null;
|
|
}
|
|
|
|
$lock = $this->locks[$keyStr];
|
|
|
|
if ($lock['expires_at'] <= $now) {
|
|
return null; // Expired
|
|
}
|
|
|
|
return [
|
|
'lock_key' => $keyStr,
|
|
'worker_id' => $lock['worker_id'],
|
|
'acquired_at' => date('Y-m-d H:i:s', $lock['acquired_at']),
|
|
'expires_at' => date('Y-m-d H:i:s', $lock['expires_at']),
|
|
'ttl_seconds' => $lock['expires_at'] - $now,
|
|
];
|
|
}
|
|
|
|
public function acquireWithTimeout(LockKey $key, WorkerId $workerId, Duration $ttl, Duration $timeout): bool
|
|
{
|
|
$startTime = microtime(true);
|
|
$timeoutSeconds = $timeout->toSeconds();
|
|
|
|
while ((microtime(true) - $startTime) < $timeoutSeconds) {
|
|
if ($this->acquire($key, $workerId, $ttl)) {
|
|
return true;
|
|
}
|
|
usleep(100000); // 100ms
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
public function releaseAllWorkerLocks(WorkerId $workerId): int
|
|
{
|
|
$workerIdStr = $workerId->toString();
|
|
$released = 0;
|
|
|
|
foreach ($this->locks as $key => $lock) {
|
|
if ($lock['worker_id'] === $workerIdStr) {
|
|
unset($this->locks[$key]);
|
|
$released++;
|
|
}
|
|
}
|
|
|
|
return $released;
|
|
}
|
|
|
|
public function cleanupExpiredLocks(): int
|
|
{
|
|
$now = time();
|
|
$cleaned = 0;
|
|
|
|
foreach ($this->locks as $key => $lock) {
|
|
if ($lock['expires_at'] <= $now) {
|
|
unset($this->locks[$key]);
|
|
$cleaned++;
|
|
}
|
|
}
|
|
|
|
return $cleaned;
|
|
}
|
|
|
|
public function getLockStatistics(): array
|
|
{
|
|
$now = time();
|
|
$activeLocks = 0;
|
|
$expiredLocks = 0;
|
|
$workers = [];
|
|
|
|
foreach ($this->locks as $lock) {
|
|
if ($lock['expires_at'] > $now) {
|
|
$activeLocks++;
|
|
$workers[$lock['worker_id']] = true;
|
|
} else {
|
|
$expiredLocks++;
|
|
}
|
|
}
|
|
|
|
return [
|
|
'total_locks' => count($this->locks),
|
|
'active_locks' => $activeLocks,
|
|
'expired_locks' => $expiredLocks,
|
|
'unique_workers' => count($workers),
|
|
'avg_ttl_seconds' => $activeLocks > 0 ?
|
|
array_sum(array_map(fn ($lock) => $lock['ttl'], array_filter($this->locks, fn ($lock) => $lock['expires_at'] > $now))) / $activeLocks
|
|
: 0,
|
|
];
|
|
}
|
|
};
|
|
});
|
|
|
|
describe('Basic Lock Operations', function () {
|
|
it('can acquire and release locks', function () {
|
|
$lockKey = LockKey::fromString('test-lock');
|
|
$workerId = WorkerId::fromString('worker-1');
|
|
$ttl = Duration::fromSeconds(300);
|
|
|
|
// Initially lock should not exist
|
|
expect($this->distributedLock->exists($lockKey))->toBeFalse();
|
|
|
|
// Acquire lock
|
|
$acquired = $this->distributedLock->acquire($lockKey, $workerId, $ttl);
|
|
expect($acquired)->toBeTrue();
|
|
|
|
// Lock should now exist
|
|
expect($this->distributedLock->exists($lockKey))->toBeTrue();
|
|
|
|
// Get lock info
|
|
$info = $this->distributedLock->getLockInfo($lockKey);
|
|
expect($info)->not->toBeNull();
|
|
expect($info['worker_id'])->toBe('worker-1');
|
|
expect($info['ttl_seconds'])->toBeGreaterThan(299);
|
|
|
|
// Release lock
|
|
$released = $this->distributedLock->release($lockKey, $workerId);
|
|
expect($released)->toBeTrue();
|
|
|
|
// Lock should no longer exist
|
|
expect($this->distributedLock->exists($lockKey))->toBeFalse();
|
|
});
|
|
|
|
it('prevents double acquisition of same lock', function () {
|
|
$lockKey = LockKey::fromString('exclusive-lock');
|
|
$worker1 = WorkerId::fromString('worker-1');
|
|
$worker2 = WorkerId::fromString('worker-2');
|
|
$ttl = Duration::fromSeconds(300);
|
|
|
|
// Worker 1 acquires lock
|
|
$acquired1 = $this->distributedLock->acquire($lockKey, $worker1, $ttl);
|
|
expect($acquired1)->toBeTrue();
|
|
|
|
// Worker 2 cannot acquire same lock
|
|
$acquired2 = $this->distributedLock->acquire($lockKey, $worker2, $ttl);
|
|
expect($acquired2)->toBeFalse();
|
|
|
|
// Worker 1 cannot acquire it again
|
|
$acquiredAgain = $this->distributedLock->acquire($lockKey, $worker1, $ttl);
|
|
expect($acquiredAgain)->toBeFalse();
|
|
});
|
|
|
|
it('only allows lock owner to release lock', function () {
|
|
$lockKey = LockKey::fromString('owner-lock');
|
|
$owner = WorkerId::fromString('owner-worker');
|
|
$other = WorkerId::fromString('other-worker');
|
|
$ttl = Duration::fromSeconds(300);
|
|
|
|
// Owner acquires lock
|
|
$this->distributedLock->acquire($lockKey, $owner, $ttl);
|
|
|
|
// Other worker cannot release it
|
|
$released = $this->distributedLock->release($lockKey, $other);
|
|
expect($released)->toBeFalse();
|
|
|
|
// Lock should still exist
|
|
expect($this->distributedLock->exists($lockKey))->toBeTrue();
|
|
|
|
// Owner can release it
|
|
$released = $this->distributedLock->release($lockKey, $owner);
|
|
expect($released)->toBeTrue();
|
|
|
|
// Lock should no longer exist
|
|
expect($this->distributedLock->exists($lockKey))->toBeFalse();
|
|
});
|
|
});
|
|
|
|
describe('Lock Extension', function () {
|
|
it('can extend lock TTL', function () {
|
|
$lockKey = LockKey::fromString('extend-test');
|
|
$workerId = WorkerId::fromString('worker-1');
|
|
$initialTtl = Duration::fromSeconds(300);
|
|
$extendedTtl = Duration::fromSeconds(600);
|
|
|
|
// Acquire lock
|
|
$this->distributedLock->acquire($lockKey, $workerId, $initialTtl);
|
|
|
|
$initialInfo = $this->distributedLock->getLockInfo($lockKey);
|
|
expect($initialInfo['ttl_seconds'])->toBeGreaterThan(299);
|
|
|
|
// Extend lock
|
|
$extended = $this->distributedLock->extend($lockKey, $workerId, $extendedTtl);
|
|
expect($extended)->toBeTrue();
|
|
|
|
$extendedInfo = $this->distributedLock->getLockInfo($lockKey);
|
|
expect($extendedInfo['ttl_seconds'])->toBeGreaterThan(599);
|
|
});
|
|
|
|
it('only allows lock owner to extend lock', function () {
|
|
$lockKey = LockKey::fromString('extend-owner-test');
|
|
$owner = WorkerId::fromString('owner-worker');
|
|
$other = WorkerId::fromString('other-worker');
|
|
$ttl = Duration::fromSeconds(300);
|
|
|
|
// Owner acquires lock
|
|
$this->distributedLock->acquire($lockKey, $owner, $ttl);
|
|
|
|
// Other worker cannot extend it
|
|
$extended = $this->distributedLock->extend($lockKey, $other, $ttl);
|
|
expect($extended)->toBeFalse();
|
|
|
|
// Owner can extend it
|
|
$extended = $this->distributedLock->extend($lockKey, $owner, $ttl);
|
|
expect($extended)->toBeTrue();
|
|
});
|
|
|
|
it('cannot extend non-existent lock', function () {
|
|
$lockKey = LockKey::fromString('non-existent-lock');
|
|
$workerId = WorkerId::fromString('worker-1');
|
|
$ttl = Duration::fromSeconds(300);
|
|
|
|
$extended = $this->distributedLock->extend($lockKey, $workerId, $ttl);
|
|
expect($extended)->toBeFalse();
|
|
});
|
|
});
|
|
|
|
describe('Lock Timeout and Acquisition', function () {
|
|
it('can acquire lock with timeout', function () {
|
|
$lockKey = LockKey::fromString('timeout-test');
|
|
$workerId = WorkerId::fromString('worker-1');
|
|
$ttl = Duration::fromSeconds(300);
|
|
$timeout = Duration::fromSeconds(1);
|
|
|
|
// Should acquire immediately since lock doesn't exist
|
|
$start = microtime(true);
|
|
$acquired = $this->distributedLock->acquireWithTimeout($lockKey, $workerId, $ttl, $timeout);
|
|
$elapsed = microtime(true) - $start;
|
|
|
|
expect($acquired)->toBeTrue();
|
|
expect($elapsed)->toBeLessThan(0.1); // Should be immediate
|
|
});
|
|
|
|
it('times out when lock is held by another worker', function () {
|
|
$lockKey = LockKey::fromString('timeout-fail-test');
|
|
$worker1 = WorkerId::fromString('worker-1');
|
|
$worker2 = WorkerId::fromString('worker-2');
|
|
$ttl = Duration::fromSeconds(300);
|
|
$timeout = Duration::fromSeconds(0.5);
|
|
|
|
// Worker 1 acquires lock
|
|
$this->distributedLock->acquire($lockKey, $worker1, $ttl);
|
|
|
|
// Worker 2 tries to acquire with timeout
|
|
$start = microtime(true);
|
|
$acquired = $this->distributedLock->acquireWithTimeout($lockKey, $worker2, $ttl, $timeout);
|
|
$elapsed = microtime(true) - $start;
|
|
|
|
expect($acquired)->toBeFalse();
|
|
expect($elapsed)->toBeGreaterThan(0.4); // Should have waited
|
|
expect($elapsed)->toBeLessThan(0.7); // But not too long
|
|
});
|
|
});
|
|
|
|
describe('Bulk Operations', function () {
|
|
it('can release all locks for a worker', function () {
|
|
$worker1 = WorkerId::fromString('worker-1');
|
|
$worker2 = WorkerId::fromString('worker-2');
|
|
$ttl = Duration::fromSeconds(300);
|
|
|
|
// Worker 1 acquires multiple locks
|
|
$lock1 = LockKey::fromString('lock-1');
|
|
$lock2 = LockKey::fromString('lock-2');
|
|
$lock3 = LockKey::fromString('lock-3');
|
|
|
|
$this->distributedLock->acquire($lock1, $worker1, $ttl);
|
|
$this->distributedLock->acquire($lock2, $worker1, $ttl);
|
|
$this->distributedLock->acquire($lock3, $worker2, $ttl); // Different worker
|
|
|
|
// Verify locks exist
|
|
expect($this->distributedLock->exists($lock1))->toBeTrue();
|
|
expect($this->distributedLock->exists($lock2))->toBeTrue();
|
|
expect($this->distributedLock->exists($lock3))->toBeTrue();
|
|
|
|
// Release all worker1 locks
|
|
$released = $this->distributedLock->releaseAllWorkerLocks($worker1);
|
|
expect($released)->toBe(2);
|
|
|
|
// Worker1 locks should be gone, worker2 lock should remain
|
|
expect($this->distributedLock->exists($lock1))->toBeFalse();
|
|
expect($this->distributedLock->exists($lock2))->toBeFalse();
|
|
expect($this->distributedLock->exists($lock3))->toBeTrue();
|
|
});
|
|
|
|
it('can cleanup expired locks', function () {
|
|
$lockKey = LockKey::fromString('expiring-lock');
|
|
$workerId = WorkerId::fromString('worker-1');
|
|
$shortTtl = Duration::fromSeconds(1);
|
|
|
|
// Acquire lock with short TTL
|
|
$this->distributedLock->acquire($lockKey, $workerId, $shortTtl);
|
|
expect($this->distributedLock->exists($lockKey))->toBeTrue();
|
|
|
|
// Wait for expiration
|
|
sleep(2);
|
|
|
|
// Lock should appear expired but still be in storage
|
|
expect($this->distributedLock->exists($lockKey))->toBeFalse();
|
|
|
|
// Cleanup should remove expired locks
|
|
$cleaned = $this->distributedLock->cleanupExpiredLocks();
|
|
expect($cleaned)->toBe(1);
|
|
});
|
|
});
|
|
|
|
describe('Lock Statistics', function () {
|
|
it('provides accurate lock statistics', function () {
|
|
$worker1 = WorkerId::fromString('worker-1');
|
|
$worker2 = WorkerId::fromString('worker-2');
|
|
$ttl = Duration::fromSeconds(300);
|
|
|
|
// Initially no locks
|
|
$stats = $this->distributedLock->getLockStatistics();
|
|
expect($stats['total_locks'])->toBe(0);
|
|
expect($stats['active_locks'])->toBe(0);
|
|
expect($stats['unique_workers'])->toBe(0);
|
|
|
|
// Add some locks
|
|
$this->distributedLock->acquire(LockKey::fromString('lock-1'), $worker1, $ttl);
|
|
$this->distributedLock->acquire(LockKey::fromString('lock-2'), $worker1, $ttl);
|
|
$this->distributedLock->acquire(LockKey::fromString('lock-3'), $worker2, $ttl);
|
|
|
|
$stats = $this->distributedLock->getLockStatistics();
|
|
expect($stats['total_locks'])->toBe(3);
|
|
expect($stats['active_locks'])->toBe(3);
|
|
expect($stats['unique_workers'])->toBe(2);
|
|
expect($stats['avg_ttl_seconds'])->toBe(300.0);
|
|
});
|
|
|
|
it('distinguishes between active and expired locks', function () {
|
|
$workerId = WorkerId::fromString('worker-1');
|
|
$longTtl = Duration::fromSeconds(300);
|
|
$shortTtl = Duration::fromSeconds(1);
|
|
|
|
// Add active and soon-to-expire locks
|
|
$this->distributedLock->acquire(LockKey::fromString('active-lock'), $workerId, $longTtl);
|
|
$this->distributedLock->acquire(LockKey::fromString('expiring-lock'), $workerId, $shortTtl);
|
|
|
|
// Initially both active
|
|
$stats = $this->distributedLock->getLockStatistics();
|
|
expect($stats['active_locks'])->toBe(2);
|
|
expect($stats['expired_locks'])->toBe(0);
|
|
|
|
// Wait for one to expire
|
|
sleep(2);
|
|
|
|
$stats = $this->distributedLock->getLockStatistics();
|
|
expect($stats['total_locks'])->toBe(2); // Still in storage
|
|
expect($stats['active_locks'])->toBe(1); // One still active
|
|
expect($stats['expired_locks'])->toBe(1); // One expired
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Distributed Lock Integration Scenarios', function () {
|
|
|
|
beforeEach(function () {
|
|
$this->distributedLock = new class () {
|
|
private array $locks = [];
|
|
|
|
public function acquire(LockKey $key, WorkerId $workerId, Duration $ttl): bool
|
|
{
|
|
$keyStr = $key->toString();
|
|
$now = time();
|
|
if (isset($this->locks[$keyStr]) && $this->locks[$keyStr]['expires_at'] > $now) {
|
|
return false;
|
|
}
|
|
$this->locks[$keyStr] = [
|
|
'worker_id' => $workerId->toString(),
|
|
'acquired_at' => $now,
|
|
'expires_at' => $now + $ttl->toSeconds(),
|
|
];
|
|
|
|
return true;
|
|
}
|
|
|
|
public function release(LockKey $key, WorkerId $workerId): bool
|
|
{
|
|
$keyStr = $key->toString();
|
|
if (! isset($this->locks[$keyStr]) || $this->locks[$keyStr]['worker_id'] !== $workerId->toString()) {
|
|
return false;
|
|
}
|
|
unset($this->locks[$keyStr]);
|
|
|
|
return true;
|
|
}
|
|
|
|
public function exists(LockKey $key): bool
|
|
{
|
|
$keyStr = $key->toString();
|
|
|
|
return isset($this->locks[$keyStr]) && $this->locks[$keyStr]['expires_at'] > time();
|
|
}
|
|
};
|
|
|
|
$this->emailJob = new class () {
|
|
public function __construct(
|
|
public string $batchId = 'email-batch-123',
|
|
public int $recipientCount = 1000
|
|
) {
|
|
}
|
|
};
|
|
|
|
$this->reportJob = new class () {
|
|
public function __construct(
|
|
public string $reportId = 'monthly-sales-2024',
|
|
public string $resourceType = 'database'
|
|
) {
|
|
}
|
|
};
|
|
});
|
|
|
|
it('demonstrates job-level locking for email batches', function () {
|
|
$batchLockKey = LockKey::forBatch($this->emailJob->batchId);
|
|
$worker1 = WorkerId::fromString('email-worker-1');
|
|
$worker2 = WorkerId::fromString('email-worker-2');
|
|
$processingTime = Duration::fromMinutes(30);
|
|
|
|
// Worker 1 acquires lock for batch processing
|
|
$acquired = $this->distributedLock->acquire($batchLockKey, $worker1, $processingTime);
|
|
expect($acquired)->toBeTrue();
|
|
|
|
// Worker 2 cannot process the same batch
|
|
$blocked = $this->distributedLock->acquire($batchLockKey, $worker2, $processingTime);
|
|
expect($blocked)->toBeFalse();
|
|
|
|
// After processing, worker 1 releases the lock
|
|
$released = $this->distributedLock->release($batchLockKey, $worker1);
|
|
expect($released)->toBeTrue();
|
|
|
|
// Now worker 2 can acquire the lock
|
|
$nowAvailable = $this->distributedLock->acquire($batchLockKey, $worker2, $processingTime);
|
|
expect($nowAvailable)->toBeTrue();
|
|
});
|
|
|
|
it('demonstrates resource-level locking for reports', function () {
|
|
$resourceLockKey = LockKey::forResource($this->reportJob->resourceType, 'primary');
|
|
$reportWorker = WorkerId::fromString('report-worker-1');
|
|
$maintenanceWorker = WorkerId::fromString('maintenance-worker');
|
|
$reportTime = Duration::fromMinutes(15);
|
|
$maintenanceTime = Duration::fromHours(2);
|
|
|
|
// Report worker acquires database resource
|
|
$reportAcquired = $this->distributedLock->acquire($resourceLockKey, $reportWorker, $reportTime);
|
|
expect($reportAcquired)->toBeTrue();
|
|
|
|
// Maintenance worker cannot access the resource
|
|
$maintenanceBlocked = $this->distributedLock->acquire($resourceLockKey, $maintenanceWorker, $maintenanceTime);
|
|
expect($maintenanceBlocked)->toBeFalse();
|
|
|
|
// Report completes and releases resource
|
|
$reportReleased = $this->distributedLock->release($resourceLockKey, $reportWorker);
|
|
expect($reportReleased)->toBeTrue();
|
|
|
|
// Maintenance can now proceed
|
|
$maintenanceAcquired = $this->distributedLock->acquire($resourceLockKey, $maintenanceWorker, $maintenanceTime);
|
|
expect($maintenanceAcquired)->toBeTrue();
|
|
});
|
|
|
|
it('demonstrates queue-level locking for exclusive processing', function () {
|
|
$emailQueueKey = LockKey::forQueue(QueueName::fromString('email-queue'));
|
|
$priorityWorker = WorkerId::fromString('priority-worker');
|
|
$normalWorker = WorkerId::fromString('normal-worker');
|
|
$exclusiveTime = Duration::fromMinutes(5);
|
|
|
|
// Priority worker takes exclusive access to queue
|
|
$exclusiveAcquired = $this->distributedLock->acquire($emailQueueKey, $priorityWorker, $exclusiveTime);
|
|
expect($exclusiveAcquired)->toBeTrue();
|
|
|
|
// Normal worker must wait
|
|
$normalBlocked = $this->distributedLock->acquire($emailQueueKey, $normalWorker, Duration::fromMinutes(1));
|
|
expect($normalBlocked)->toBeFalse();
|
|
|
|
// After priority processing, queue becomes available
|
|
$exclusiveReleased = $this->distributedLock->release($emailQueueKey, $priorityWorker);
|
|
expect($exclusiveReleased)->toBeTrue();
|
|
|
|
$normalAcquired = $this->distributedLock->acquire($emailQueueKey, $normalWorker, Duration::fromMinutes(10));
|
|
expect($normalAcquired)->toBeTrue();
|
|
});
|
|
|
|
it('demonstrates worker-level locking for maintenance operations', function () {
|
|
$worker1 = WorkerId::fromString('maintenance-worker-1');
|
|
$worker1LockKey = LockKey::forWorker($worker1);
|
|
$maintenanceSystem = WorkerId::fromString('maintenance-system');
|
|
$maintenanceTime = Duration::fromHours(1);
|
|
|
|
// Worker is performing normal operations
|
|
expect($this->distributedLock->exists($worker1LockKey))->toBeFalse();
|
|
|
|
// Maintenance system needs to pause the worker
|
|
$maintenanceLock = $this->distributedLock->acquire($worker1LockKey, $maintenanceSystem, $maintenanceTime);
|
|
expect($maintenanceLock)->toBeTrue();
|
|
|
|
// Worker cannot proceed with new jobs while maintenance lock is held
|
|
$workerBlocked = $this->distributedLock->acquire($worker1LockKey, $worker1, Duration::fromMinutes(1));
|
|
expect($workerBlocked)->toBeFalse();
|
|
|
|
// After maintenance, worker can resume
|
|
$maintenanceReleased = $this->distributedLock->release($worker1LockKey, $maintenanceSystem);
|
|
expect($maintenanceReleased)->toBeTrue();
|
|
|
|
$workerResumed = $this->distributedLock->acquire($worker1LockKey, $worker1, Duration::fromMinutes(30));
|
|
expect($workerResumed)->toBeTrue();
|
|
});
|
|
|
|
it('demonstrates hierarchical locking patterns', function () {
|
|
$systemLock = LockKey::fromString('system.maintenance');
|
|
$queueLock = LockKey::fromString('system.queue.email');
|
|
$jobLock = LockKey::fromString('system.queue.email.job-123');
|
|
|
|
$maintenanceWorker = WorkerId::fromString('maintenance-worker');
|
|
$queueWorker = WorkerId::fromString('queue-worker');
|
|
$jobWorker = WorkerId::fromString('job-worker');
|
|
|
|
// System-wide maintenance lock
|
|
$systemAcquired = $this->distributedLock->acquire($systemLock, $maintenanceWorker, Duration::fromHours(1));
|
|
expect($systemAcquired)->toBeTrue();
|
|
|
|
// Queue-level operations should be blocked during system maintenance
|
|
$queueBlocked = $this->distributedLock->acquire($queueLock, $queueWorker, Duration::fromMinutes(30));
|
|
expect($queueBlocked)->toBeFalse();
|
|
|
|
// Job-level operations should also be blocked
|
|
$jobBlocked = $this->distributedLock->acquire($jobLock, $jobWorker, Duration::fromMinutes(5));
|
|
expect($jobBlocked)->toBeFalse();
|
|
|
|
// After system maintenance
|
|
$this->distributedLock->release($systemLock, $maintenanceWorker);
|
|
|
|
// Lower-level operations can proceed
|
|
$queueNowAcquired = $this->distributedLock->acquire($queueLock, $queueWorker, Duration::fromMinutes(30));
|
|
expect($queueNowAcquired)->toBeTrue();
|
|
});
|
|
});
|