diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index fba86ed25..f82634546 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -1709,6 +1709,41 @@ function () use ($collection, $chunk): array { * @throws DuplicateException * @throws StructureException */ + /** + * Whether an update changes nothing but unbounded increments and decrements, so it can be applied + * without a locked read. $id and $permissions may be present unchanged, as Appwrite always sends them; + * unchanged permissions are confirmed once the stored document is read. + */ + private function changesOnlyOperators(Document $collection, string $id, Document $document): bool + { + if ( + $collection->getId() === self::METADATA + || $this->requestTimestamp()->get() !== null + || ! $this->adapter->supports(Capability::UpdateLock) + ) { + return false; + } + + $operators = 0; + foreach ($document->getArrayCopy() as $key => $value) { + if ($key === Document::PERMISSIONS || $key === Document::COLLECTION || ($key === Document::ID && $value === $id)) { + continue; + } + // Only unbounded increments: a bound is validated against the stored value, which is read + // without the lock here and may already be stale. + if ( + ! $value instanceof Operator + || ! \in_array($value->getMethod(), [OperatorType::Increment, OperatorType::Decrement], true) + || \count($value->getValues()) > 1 + ) { + return false; + } + $operators++; + } + + return $operators > 0; + } + public function updateDocument(string $collection, string $id, Document $document): Document { if (! $id) { @@ -1722,13 +1757,31 @@ public function updateDocument(string $collection, string $id, Document $documen $cacheTarget = $collection->getId() === self::METADATA ? new Document([Document::ID => $id, Document::COLLECTION => self::METADATA]) : $collection->getId(); - $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $preparation, $newUpdatedAt, &$hasOperators) { - $old = $this->authorization->skip(fn () => $this->silent( - fn () => $this->skipQueryHooks(fn () => $this->readDocument($collection, $id, forUpdate: true)) + $operatorsOnly = $this->changesOnlyOperators($collection, $id, $document); + $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $preparation, $newUpdatedAt, &$hasOperators, $operatorsOnly) { + $read = fn (bool $lock) => $this->authorization->skip(fn () => $this->silent( + fn () => $this->skipQueryHooks(fn () => $this->readDocument($collection, $id, forUpdate: $lock)) )); + // An update made only of operators writes nothing it read, so it does not need the row lock + // until its UPDATE takes it. Waiting on a locked read here serialises every concurrent + // increment of a hot row behind the whole of the previous one's transaction. + $old = $read(! $operatorsOnly); + if ($operatorsOnly && $document->offsetExists(Document::PERMISSIONS)) { + $stored = $old->getPermissions(); + $given = $document->getPermissions(); + \sort($stored); + \sort($given); + if ($stored !== $given) { + $operatorsOnly = false; + $old = $read(true); + } + } if ($old->isEmpty()) { return new Document(); } + $operatorKeys = $operatorsOnly + ? \array_keys(\array_filter($document->getArrayCopy(), fn ($value) => Operator::isOperator($value))) + : []; $document = $this->prepareNestedCopy($collection, $document, $preparation); $time = DateTime::nowAfter($old->getUpdatedAt() ?: null); @@ -1954,7 +2007,17 @@ public function updateDocument(string $collection, string $id, Document $documen $document = $this->castBefore($collection, $document); - $this->authorization->skip(fn () => $this->adapter->updateDocument($collection, $old->getId(), $document, $skipPermissionsUpdate)); + if ($operatorsOnly) { + // Write the operator columns and $updatedAt only: every other column of $document was read + // without the lock and may be stale, so writing it back would undo a concurrent update. + $changes = new Document([Document::UPDATED_AT => $document->getUpdatedAt()]); + foreach ($operatorKeys as $key) { + $changes->setAttribute($key, $document->getAttribute($key)); + } + $this->authorization->skip(fn () => $this->adapter->updateDocuments($collection, $changes, [$document], [$document->getId() => true])); + } else { + $this->authorization->skip(fn () => $this->adapter->updateDocument($collection, $old->getId(), $document, $skipPermissionsUpdate)); + } $document = $this->castAfterDocument($collection, $document); @@ -1968,7 +2031,7 @@ public function updateDocument(string $collection, string $id, Document $documen $this->queueDocumentPurge($collection->getId(), $purgedId); } - if ($hasOperators) { + if ($hasOperators && ! $operatorsOnly) { $refetched = $this->refetchDocuments($collection, [$document]); $document = $refetched[0]; } @@ -1980,6 +2043,12 @@ public function updateDocument(string $collection, string $id, Document $documen return $document; } + // Read the operators' results once the row lock is released. The values include this update and + // any that committed after it, never fewer. + if ($operatorsOnly) { + $document = $this->refetchDocuments($collection, [$document])[0]; + } + $hook = $this->relationshipHook; if ($hook !== null && ! $hook->isInBatchPopulation() && $hook->isEnabled()) { $documents = $this->silent(fn () => $this->skipQueryHooks(fn () => $hook->populateDocuments([$document], $collection, $hook->getFetchDepth()))); diff --git a/tests/e2e/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php index 5eaf3d034..5bffd9923 100644 --- a/tests/e2e/Adapter/MySQLTest.php +++ b/tests/e2e/Adapter/MySQLTest.php @@ -7,11 +7,20 @@ use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Attribute; +use Utopia\Database\Collection; use Utopia\Database\Database; +use Utopia\Database\Document; use Utopia\Database\Exception; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Exception\Limit; +use Utopia\Database\Operator; use Utopia\Database\PDO; +use Utopia\Database\Permission; +use Utopia\Database\PermissionType; +use Utopia\Database\Role; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\Authorization\Input; class MySQLTest extends Base { @@ -78,6 +87,106 @@ protected function deleteColumn(string $collection, string $column): bool return true; } + /** + * A concurrent write to another column, landing after the update read the row and before it wrote, is + * neither blocked by the update nor undone by it: an update of operators only holds no lock while reading + * and writes back only its own columns. + */ + public function testOperatorOnlyUpdateLeavesConcurrentWriteToAnotherColumn(): void + { + [$updated, $row, $blocked] = $this->incrementWhileAnotherColumnIsWritten(Operator::increment(1)); + + $this->assertFalse($blocked, 'The concurrent write waited for the row lock'); + $this->assertSame(1, $updated->getAttribute('opens')); + $this->assertSame(['opens' => 1, 'status' => 'concurrent'], $row); + } + + /** + * A bounded increment is validated against the stored value, so it still reads the row under the lock: + * a concurrent write cannot land between that check and the write. + */ + public function testBoundedIncrementKeepsTheRowLockedFromReadToWrite(): void + { + [$updated, $row, $blocked] = $this->incrementWhileAnotherColumnIsWritten(Operator::increment(1, 10)); + + $this->assertTrue($blocked, 'The concurrent write landed between the bound check and the write'); + $this->assertSame(1, $updated->getAttribute('opens')); + $this->assertSame(['opens' => 1, 'status' => 'before'], $row); + } + + /** + * Increment a row's `opens` while a second connection writes its `status` between the update's read and + * its write. + * + * @return array{Document, array{opens: int, status: string}, bool} The updated document, the stored row, + * and whether the other write hit a lock wait timeout + */ + private function incrementWhileAnotherColumnIsWritten(Operator $increment): array + { + $database = $this->getDatabase(); + $collection = 'hot_row_'.\uniqid(); + $database->createCollection(Collection::create( + id: $collection, + attributes: [Attribute::integer(key: 'opens'), Attribute::string(key: 'status', size: 32)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + documentSecurity: false, + )); + $database->createDocument($collection, new Document([Document::ID => 'row', 'opens' => 0, 'status' => 'before'])); + + $other = new PDO('mysql:host=mysql;port=3307;charset=utf8mb4', 'root', 'password', [\PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION]); + $other->exec('SET SESSION innodb_lock_wait_timeout = 1'); + $table = '`'.$database->getDatabase().'`.`'.$database->getNamespace().'_'.$collection.'`'; + + $between = new class () extends Authorization { + public ?\Closure $write = null; + + #[\Override] + public function isValid(mixed $value): bool + { + if ($this->write !== null && $value instanceof Input && $value->getAction() === PermissionType::Update->value) { + ($this->write)(); + $this->write = null; + } + + return parent::isValid($value); + } + }; + $blocked = false; + $between->write = function () use ($other, $table, &$blocked): void { + try { + $other->exec("UPDATE {$table} SET `status` = 'concurrent' WHERE `_uid` = 'row'"); + } catch (\PDOException $error) { + $blocked = ($error->errorInfo[1] ?? null) === 1205; + if (! $blocked) { + throw $error; + } + } + }; + + $authorization = $database->getAuthorization(); + $database->setAuthorization($between); + try { + $updated = $database->updateDocument($collection, 'row', new Document([ + Document::ID => 'row', + Document::PERMISSIONS => [], + 'opens' => $increment, + ])); + } finally { + $database->setAuthorization($authorization); + } + + $this->assertNull($between->write, 'The concurrent write never ran'); + $statement = $other->query("SELECT `opens`, `status` FROM {$table} WHERE `_uid` = 'row'"); + $this->assertNotFalse($statement); + $row = $statement->fetch(\PDO::FETCH_ASSOC); + $this->assertIsArray($row); + $this->assertIsInt($row['opens']); + $this->assertIsString($row['status']); + $database->deleteCollection($collection); + + return [$updated, ['opens' => $row['opens'], 'status' => $row['status']], $blocked]; + } + #[\Override] protected function deleteIndex(string $collection, string $index): bool {