From a3b1cad0f0f04e88b567d3408b90a567b9f39bdb Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:57:55 +0000 Subject: [PATCH 1/2] Update a document of operators only without a locked read An update made of operators alone, such as a counter increment, read the row with SELECT ... FOR UPDATE and wrote every column back. Concurrent increments of a hot row queued behind the whole of each other's transaction: the locked read, validation, the write, the re-read of the operators' results and the cache purge. Such an update now reads the row without the lock, writes only its operator columns and $updatedAt (so nothing read without the lock is written back), and re-reads the results after commit. The UPDATE itself still takes the row lock, so increments stay atomic. It applies only when every changed attribute is an operator on a non-relationship attribute, $id and $permissions are absent or unchanged, no request timestamp is set, and the adapter locks reads for update. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Documents.php | 80 ++++++++++++++++++++++++++++++-- tests/e2e/Adapter/MySQLTest.php | 67 ++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 5 deletions(-) diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index fba86ed25..9ecd58d2c 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -1709,6 +1709,42 @@ function () use ($collection, $chunk): array { * @throws DuplicateException * @throws StructureException */ + /** + * Whether an update changes nothing but operator attributes, such as an increment, so it can be applied + * without a locked read. $id and $permissions may be present unchanged, as Appwrite always sends them; + * unchanged permissions are confirmed once the stored document is read. + */ + private function changesOnlyOperators(Document $collection, string $id, Document $document): bool + { + if ( + $collection->getId() === self::METADATA + || $this->requestTimestamp()->get() !== null + || ! $this->adapter->supports(Capability::UpdateLock) + ) { + return false; + } + + $relationships = []; + foreach ($collection->attributes() as $attribute) { + if ($attribute->relationship !== null) { + $relationships[$attribute->key] = true; + } + } + + $operators = 0; + foreach ($document->getArrayCopy() as $key => $value) { + if ($key === Document::PERMISSIONS || $key === Document::COLLECTION || ($key === Document::ID && $value === $id)) { + continue; + } + if (! Operator::isOperator($value) || isset($relationships[$key])) { + return false; + } + $operators++; + } + + return $operators > 0; + } + public function updateDocument(string $collection, string $id, Document $document): Document { if (! $id) { @@ -1722,13 +1758,31 @@ public function updateDocument(string $collection, string $id, Document $documen $cacheTarget = $collection->getId() === self::METADATA ? new Document([Document::ID => $id, Document::COLLECTION => self::METADATA]) : $collection->getId(); - $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $preparation, $newUpdatedAt, &$hasOperators) { - $old = $this->authorization->skip(fn () => $this->silent( - fn () => $this->skipQueryHooks(fn () => $this->readDocument($collection, $id, forUpdate: true)) + $operatorsOnly = $this->changesOnlyOperators($collection, $id, $document); + $document = $this->withMutation(Event::DocumentUpdate, $cacheTarget, function () use ($collection, $id, $document, $preparation, $newUpdatedAt, &$hasOperators, $operatorsOnly) { + $read = fn (bool $lock) => $this->authorization->skip(fn () => $this->silent( + fn () => $this->skipQueryHooks(fn () => $this->readDocument($collection, $id, forUpdate: $lock)) )); + // An update made only of operators writes nothing it read, so it does not need the row lock + // until its UPDATE takes it. Waiting on a locked read here serialises every concurrent + // increment of a hot row behind the whole of the previous one's transaction. + $old = $read(! $operatorsOnly); + if ($operatorsOnly && $document->offsetExists(Document::PERMISSIONS)) { + $stored = $old->getPermissions(); + $given = $document->getPermissions(); + \sort($stored); + \sort($given); + if ($stored !== $given) { + $operatorsOnly = false; + $old = $read(true); + } + } if ($old->isEmpty()) { return new Document(); } + $operatorKeys = $operatorsOnly + ? \array_keys(\array_filter($document->getArrayCopy(), fn ($value) => Operator::isOperator($value))) + : []; $document = $this->prepareNestedCopy($collection, $document, $preparation); $time = DateTime::nowAfter($old->getUpdatedAt() ?: null); @@ -1954,7 +2008,17 @@ public function updateDocument(string $collection, string $id, Document $documen $document = $this->castBefore($collection, $document); - $this->authorization->skip(fn () => $this->adapter->updateDocument($collection, $old->getId(), $document, $skipPermissionsUpdate)); + if ($operatorsOnly) { + // Write the operator columns and $updatedAt only: every other column of $document was read + // without the lock and may be stale, so writing it back would undo a concurrent update. + $changes = new Document([Document::UPDATED_AT => $document->getUpdatedAt()]); + foreach ($operatorKeys as $key) { + $changes->setAttribute($key, $document->getAttribute($key)); + } + $this->authorization->skip(fn () => $this->adapter->updateDocuments($collection, $changes, [$document], [$document->getId() => true])); + } else { + $this->authorization->skip(fn () => $this->adapter->updateDocument($collection, $old->getId(), $document, $skipPermissionsUpdate)); + } $document = $this->castAfterDocument($collection, $document); @@ -1968,7 +2032,7 @@ public function updateDocument(string $collection, string $id, Document $documen $this->queueDocumentPurge($collection->getId(), $purgedId); } - if ($hasOperators) { + if ($hasOperators && ! $operatorsOnly) { $refetched = $this->refetchDocuments($collection, [$document]); $document = $refetched[0]; } @@ -1980,6 +2044,12 @@ public function updateDocument(string $collection, string $id, Document $documen return $document; } + // Read the operators' results once the row lock is released. The values include this update and + // any that committed after it, never fewer. + if ($operatorsOnly) { + $document = $this->refetchDocuments($collection, [$document])[0]; + } + $hook = $this->relationshipHook; if ($hook !== null && ! $hook->isInBatchPopulation() && $hook->isEnabled()) { $documents = $this->silent(fn () => $this->skipQueryHooks(fn () => $hook->populateDocuments([$document], $collection, $hook->getFetchDepth()))); diff --git a/tests/e2e/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php index 5eaf3d034..bc920c1c5 100644 --- a/tests/e2e/Adapter/MySQLTest.php +++ b/tests/e2e/Adapter/MySQLTest.php @@ -7,11 +7,20 @@ use Utopia\Cache\Adapter\Redis as RedisAdapter; use Utopia\Cache\Cache; use Utopia\Database\Adapter\MySQL; +use Utopia\Database\Attribute; +use Utopia\Database\Collection; use Utopia\Database\Database; +use Utopia\Database\Document; use Utopia\Database\Exception; use Utopia\Database\Exception\Duplicate; use Utopia\Database\Exception\Limit; +use Utopia\Database\Operator; use Utopia\Database\PDO; +use Utopia\Database\Permission; +use Utopia\Database\PermissionType; +use Utopia\Database\Role; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\Authorization\Input; class MySQLTest extends Base { @@ -78,6 +87,64 @@ protected function deleteColumn(string $collection, string $column): bool return true; } + /** + * A concurrent write to another column, landing after the update read the row and before it wrote, is + * neither blocked by the update nor undone by it: an update of operators only holds no lock while reading + * and writes back only its own columns. + */ + public function testOperatorOnlyUpdateLeavesConcurrentWriteToAnotherColumn(): void + { + $database = $this->getDatabase(); + $collection = 'hot_row_'.\uniqid(); + $database->createCollection(Collection::create( + id: $collection, + attributes: [Attribute::integer(key: 'opens'), Attribute::string(key: 'status', size: 32)], + permissions: [Permission::create(Role::any()), Permission::read(Role::any()), Permission::update(Role::any())], + documentSecurity: false, + )); + $database->createDocument($collection, new Document([Document::ID => 'row', 'opens' => 0, 'status' => 'before'])); + + $other = new PDO('mysql:host=mysql;port=3307;charset=utf8mb4', 'root', 'password', [\PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION]); + $other->exec('SET SESSION innodb_lock_wait_timeout = 1'); + $table = '`'.$database->getDatabase().'`.`'.$database->getNamespace().'_'.$collection.'`'; + + $between = new class () extends Authorization { + public ?\Closure $write = null; + + #[\Override] + public function isValid(mixed $value): bool + { + if ($this->write !== null && $value instanceof Input && $value->getAction() === PermissionType::Update->value) { + ($this->write)(); + $this->write = null; + } + + return parent::isValid($value); + } + }; + $between->write = fn () => $other->exec("UPDATE {$table} SET `status` = 'concurrent' WHERE `_uid` = 'row'"); + + $authorization = $database->getAuthorization(); + $database->setAuthorization($between); + try { + $updated = $database->updateDocument($collection, 'row', new Document([ + Document::ID => 'row', + Document::PERMISSIONS => [], + 'opens' => Operator::increment(1), + ])); + } finally { + $database->setAuthorization($authorization); + } + + $this->assertNull($between->write, 'The concurrent write never ran'); + $this->assertSame(1, $updated->getAttribute('opens')); + $statement = $other->query("SELECT `opens`, `status` FROM {$table} WHERE `_uid` = 'row'"); + $this->assertNotFalse($statement); + $this->assertSame([['opens' => 1, 'status' => 'concurrent']], $statement->fetchAll(\PDO::FETCH_ASSOC)); + + $database->deleteCollection($collection); + } + #[\Override] protected function deleteIndex(string $collection, string $index): bool { From c1bac8cbfb58cefc33a2507db907b7b4a5e0b589 Mon Sep 17 00:00:00 2001 From: loks0n <22452787+loks0n@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:14:23 +0000 Subject: [PATCH 2/2] Keep bounded and non-numeric operators on the locked path A bound such as increment(1, max) is validated against the stored value, which the fast path reads without the lock, so a concurrent change could slip between the check and the write. Only unbounded increments and decrements now skip the locked read. They only apply to numeric attributes, so the relationship check goes. Co-Authored-By: Claude Opus 5.5 --- src/Database/Trait/Documents.php | 17 +++++------ tests/e2e/Adapter/MySQLTest.php | 52 +++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/src/Database/Trait/Documents.php b/src/Database/Trait/Documents.php index 9ecd58d2c..f82634546 100644 --- a/src/Database/Trait/Documents.php +++ b/src/Database/Trait/Documents.php @@ -1710,7 +1710,7 @@ function () use ($collection, $chunk): array { * @throws StructureException */ /** - * Whether an update changes nothing but operator attributes, such as an increment, so it can be applied + * Whether an update changes nothing but unbounded increments and decrements, so it can be applied * without a locked read. $id and $permissions may be present unchanged, as Appwrite always sends them; * unchanged permissions are confirmed once the stored document is read. */ @@ -1724,19 +1724,18 @@ private function changesOnlyOperators(Document $collection, string $id, Document return false; } - $relationships = []; - foreach ($collection->attributes() as $attribute) { - if ($attribute->relationship !== null) { - $relationships[$attribute->key] = true; - } - } - $operators = 0; foreach ($document->getArrayCopy() as $key => $value) { if ($key === Document::PERMISSIONS || $key === Document::COLLECTION || ($key === Document::ID && $value === $id)) { continue; } - if (! Operator::isOperator($value) || isset($relationships[$key])) { + // Only unbounded increments: a bound is validated against the stored value, which is read + // without the lock here and may already be stale. + if ( + ! $value instanceof Operator + || ! \in_array($value->getMethod(), [OperatorType::Increment, OperatorType::Decrement], true) + || \count($value->getValues()) > 1 + ) { return false; } $operators++; diff --git a/tests/e2e/Adapter/MySQLTest.php b/tests/e2e/Adapter/MySQLTest.php index bc920c1c5..5bffd9923 100644 --- a/tests/e2e/Adapter/MySQLTest.php +++ b/tests/e2e/Adapter/MySQLTest.php @@ -93,6 +93,35 @@ protected function deleteColumn(string $collection, string $column): bool * and writes back only its own columns. */ public function testOperatorOnlyUpdateLeavesConcurrentWriteToAnotherColumn(): void + { + [$updated, $row, $blocked] = $this->incrementWhileAnotherColumnIsWritten(Operator::increment(1)); + + $this->assertFalse($blocked, 'The concurrent write waited for the row lock'); + $this->assertSame(1, $updated->getAttribute('opens')); + $this->assertSame(['opens' => 1, 'status' => 'concurrent'], $row); + } + + /** + * A bounded increment is validated against the stored value, so it still reads the row under the lock: + * a concurrent write cannot land between that check and the write. + */ + public function testBoundedIncrementKeepsTheRowLockedFromReadToWrite(): void + { + [$updated, $row, $blocked] = $this->incrementWhileAnotherColumnIsWritten(Operator::increment(1, 10)); + + $this->assertTrue($blocked, 'The concurrent write landed between the bound check and the write'); + $this->assertSame(1, $updated->getAttribute('opens')); + $this->assertSame(['opens' => 1, 'status' => 'before'], $row); + } + + /** + * Increment a row's `opens` while a second connection writes its `status` between the update's read and + * its write. + * + * @return array{Document, array{opens: int, status: string}, bool} The updated document, the stored row, + * and whether the other write hit a lock wait timeout + */ + private function incrementWhileAnotherColumnIsWritten(Operator $increment): array { $database = $this->getDatabase(); $collection = 'hot_row_'.\uniqid(); @@ -122,7 +151,17 @@ public function isValid(mixed $value): bool return parent::isValid($value); } }; - $between->write = fn () => $other->exec("UPDATE {$table} SET `status` = 'concurrent' WHERE `_uid` = 'row'"); + $blocked = false; + $between->write = function () use ($other, $table, &$blocked): void { + try { + $other->exec("UPDATE {$table} SET `status` = 'concurrent' WHERE `_uid` = 'row'"); + } catch (\PDOException $error) { + $blocked = ($error->errorInfo[1] ?? null) === 1205; + if (! $blocked) { + throw $error; + } + } + }; $authorization = $database->getAuthorization(); $database->setAuthorization($between); @@ -130,19 +169,22 @@ public function isValid(mixed $value): bool $updated = $database->updateDocument($collection, 'row', new Document([ Document::ID => 'row', Document::PERMISSIONS => [], - 'opens' => Operator::increment(1), + 'opens' => $increment, ])); } finally { $database->setAuthorization($authorization); } $this->assertNull($between->write, 'The concurrent write never ran'); - $this->assertSame(1, $updated->getAttribute('opens')); $statement = $other->query("SELECT `opens`, `status` FROM {$table} WHERE `_uid` = 'row'"); $this->assertNotFalse($statement); - $this->assertSame([['opens' => 1, 'status' => 'concurrent']], $statement->fetchAll(\PDO::FETCH_ASSOC)); - + $row = $statement->fetch(\PDO::FETCH_ASSOC); + $this->assertIsArray($row); + $this->assertIsInt($row['opens']); + $this->assertIsString($row['status']); $database->deleteCollection($collection); + + return [$updated, ['opens' => $row['opens'], 'status' => $row['status']], $blocked]; } #[\Override]