From 1e34630c6bac2d3af0a6a3493718353f8afd9ad0 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 16 Nov 2026 15:30:01 +0200 Subject: [PATCH 1/5] RemoveUnknownAttributes --- src/Database/Database.php | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index 12d85006b..5e72088cb 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1960,11 +1960,24 @@ public function encode(Document $collection, Document $document, bool $applyDefa * counted as a write. encode() also calls this once it has collected the known * set while iterating the schema. * + * The metadata collection drops whatever the flag says. Its schema is declared in code, + * so a key it does not declare is never the caller's data: it is a physical column this + * version knows nothing about, which a read hands back regardless because the adapter + * selects every column. Left on the document it fails structure validation on the way + * back, and takes the schema operation with it -- so a deployment that predates a column + * could not add one to a table that already had it, which is the order a rollout wants. + * The caller's own tables still honour the flag, because a key they sent that this schema + * does not declare may well be data worth refusing rather than discarding. + * * @param array|null $known Attribute ids already collected */ protected function removeUnknownAttributes(Document $collection, Document $document, ?array $known = null): Document { - if (! $this->dropUnknownAttributes || ! $this->adapter->supports(Capability::DefinedAttributes)) { + if (! $this->adapter->supports(Capability::DefinedAttributes)) { + return $document; + } + + if (! $this->dropUnknownAttributes && $collection->getId() !== self::METADATA) { return $document; } From 656613dd283429f1946012d43501065f14da1c16 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 16 Nov 2026 16:42:47 +0200 Subject: [PATCH 2/5] Add test --- tests/e2e/Adapter/Scopes/DocumentTests.php | 66 ++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 825915476..d11ab14ef 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -9838,6 +9838,72 @@ public function testDropUnknownAttributes(): void $this->assertSame(false, $database->isDroppingUnknownAttributes()); } + /** + * A column reaches _metadata before the release that declares it, which is the order a + * rollout wants: the engine is ready before anything writes the key. The definition is + * read with SELECT *, so that column comes back on a document whose schema does not + * declare it, and the schema operation that writes the definition back would fail + * structure validation and roll its own DDL back with it. Metadata drops the key whatever + * the flag says; the caller's own tables still refuse one, because there it may be data + * they sent. + */ + public function testMetadataDropsUnknownAttributes(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + + if (! $database->getAdapter()->supports(Capability::DefinedAttributes)) { + $this->expectNotToPerformAssertions(); + + return; + } + + $database->createCollection(Collection::create(id: __FUNCTION__)); + $this->assertSame(false, $database->isDroppingUnknownAttributes()); + + // What a patch script ahead of the release writes: the physical column alone. The + // metadata schema is declared in code, so there is no stored definition to keep in step. + $database->getAdapter()->createAttribute( + Database::METADATA, + Attribute::string(key: 'unreleased', size: 1_000_000, filters: [Filter::Json]), + ); + + try { + // Drop the cached definition, so the next read is the one that brings the column back. + $database->purgeCachedDocument(Database::METADATA, __FUNCTION__); + + $this->assertSame( + 'known', + $database->createAttribute(__FUNCTION__, Attribute::string(key: 'known', size: 128))->key, + 'A schema operation failed against a metadata table holding an undeclared column' + ); + + $this->assertNull( + $database->getCollection(__FUNCTION__)->getAttribute('unreleased'), + 'The undeclared column was written back onto the definition' + ); + + // The caller's own table is untouched by this: an attribute it does not declare is + // still refused rather than discarded. + try { + $database->createDocument(__FUNCTION__, new Document([ + '$id' => 'strict', + '$permissions' => [Permission::read(Role::any())], + 'known' => 'kept', + 'unknown' => 'refused', + ])); + $this->fail('Unknown attribute was accepted on a caller-owned table'); + } catch (StructureException $e) { + $this->assertEquals('Invalid document structure: Unknown attribute: "unknown"', $e->getMessage()); + } + } finally { + $database->getAdapter()->deleteAttribute(Database::METADATA, 'unreleased'); + $database->purgeCachedDocument(Database::METADATA, __FUNCTION__); + } + + $database->deleteCollection(__FUNCTION__); + } + public function testCreateDocumentWithBigIntType(): void { $database = $this->getDatabase(); From 8a974dea15371c994ec483237fa3ce816b5419c6 Mon Sep 17 00:00:00 2001 From: fogelito Date: Mon, 16 Nov 2026 16:51:24 +0200 Subject: [PATCH 3/5] fix sql lite test --- tests/e2e/Adapter/Scopes/DocumentTests.php | 73 ++++++++++------------ 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index d11ab14ef..fa8116f3c 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -9840,12 +9840,12 @@ public function testDropUnknownAttributes(): void /** * A column reaches _metadata before the release that declares it, which is the order a - * rollout wants: the engine is ready before anything writes the key. The definition is - * read with SELECT *, so that column comes back on a document whose schema does not - * declare it, and the schema operation that writes the definition back would fail + * rollout wants: the engine is ready before anything writes the key. The definition is read + * with every column selected, so that column rides back on a document whose schema knows + * nothing about it, and the schema operation that writes the definition back would fail * structure validation and roll its own DDL back with it. Metadata drops the key whatever - * the flag says; the caller's own tables still refuse one, because there it may be data - * they sent. + * the flag says; the caller's own tables still keep it, because there it may be data they + * sent and refusing the write tells them so. */ public function testMetadataDropsUnknownAttributes(): void { @@ -9859,46 +9859,41 @@ public function testMetadataDropsUnknownAttributes(): void } $database->createCollection(Collection::create(id: __FUNCTION__)); + $this->assertSame('known', $database->createAttribute(__FUNCTION__, Attribute::string(key: 'known', size: 128))->key); $this->assertSame(false, $database->isDroppingUnknownAttributes()); - // What a patch script ahead of the release writes: the physical column alone. The - // metadata schema is declared in code, so there is no stored definition to keep in step. - $database->getAdapter()->createAttribute( - Database::METADATA, - Attribute::string(key: 'unreleased', size: 1_000_000, filters: [Filter::Json]), - ); + $metadata = $database->encode(Database::collectionDefinition(), new Document([ + '$id' => __FUNCTION__, + '$collection' => Database::METADATA, + 'name' => __FUNCTION__, + 'documentSecurity' => false, + 'unreleased' => 'dropped', + ])); - try { - // Drop the cached definition, so the next read is the one that brings the column back. - $database->purgeCachedDocument(Database::METADATA, __FUNCTION__); + $this->assertEquals(__FUNCTION__, $metadata->getAttribute('name')); + $this->assertNull($metadata->getAttribute('unreleased'), 'Metadata kept an attribute it does not declare'); - $this->assertSame( - 'known', - $database->createAttribute(__FUNCTION__, Attribute::string(key: 'known', size: 128))->key, - 'A schema operation failed against a metadata table holding an undeclared column' - ); + // The caller's own table is untouched by this: the attribute survives encode and the + // write is refused, rather than discarded on the way through. + $encoded = $database->encode($database->getCollection(__FUNCTION__), new Document([ + '$id' => 'strict', + '$collection' => __FUNCTION__, + 'known' => 'kept', + 'unknown' => 'refused', + ])); - $this->assertNull( - $database->getCollection(__FUNCTION__)->getAttribute('unreleased'), - 'The undeclared column was written back onto the definition' - ); + $this->assertEquals('refused', $encoded->getAttribute('unknown'), 'A caller-owned table dropped an attribute while dropping is disabled'); - // The caller's own table is untouched by this: an attribute it does not declare is - // still refused rather than discarded. - try { - $database->createDocument(__FUNCTION__, new Document([ - '$id' => 'strict', - '$permissions' => [Permission::read(Role::any())], - 'known' => 'kept', - 'unknown' => 'refused', - ])); - $this->fail('Unknown attribute was accepted on a caller-owned table'); - } catch (StructureException $e) { - $this->assertEquals('Invalid document structure: Unknown attribute: "unknown"', $e->getMessage()); - } - } finally { - $database->getAdapter()->deleteAttribute(Database::METADATA, 'unreleased'); - $database->purgeCachedDocument(Database::METADATA, __FUNCTION__); + try { + $database->createDocument(__FUNCTION__, new Document([ + '$id' => 'strict', + '$permissions' => [Permission::read(Role::any())], + 'known' => 'kept', + 'unknown' => 'refused', + ])); + $this->fail('Unknown attribute was accepted on a caller-owned table'); + } catch (StructureException $e) { + $this->assertEquals('Invalid document structure: Unknown attribute: "unknown"', $e->getMessage()); } $database->deleteCollection(__FUNCTION__); From 05d271b040e6112f9b9537234e9425b8c234ff30 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 17 Nov 2026 12:04:23 +0200 Subject: [PATCH 4/5] Add tests --- tests/e2e/Adapter/Scopes/DocumentTests.php | 81 ++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index fa8116f3c..87857479d 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -9899,6 +9899,87 @@ public function testMetadataDropsUnknownAttributes(): void $database->deleteCollection(__FUNCTION__); } + /** + * The rollout end to end: a patch script puts the column on _metadata before the release + * that declares it, and a schema operation then has to survive reading it back. Without the + * exemption the definition write fails structure validation and rolls its own DDL back. + * + * SQLite is left out because only the cleanup is unportable: deleteAttribute rebuilds a + * table from the stored definition row that _metadata has none of, so the column could not + * be taken off again, and it is shared with every test after this one. The flag-independent + * branch itself is covered on every adapter by testMetadataDropsUnknownAttributes. + */ + public function testMetadataDropsUnknownAttributesOnSchemaUpdate(): void + { + /** @var Database $database */ + $database = $this->getDatabase(); + $adapter = $database->getAdapter(); + + if (! $adapter->supports(Capability::DefinedAttributes)) { + $this->expectNotToPerformAssertions(); + + return; + } + + if ($adapter instanceof SQLite || $adapter instanceof Memory || $adapter instanceof Redis) { + $this->expectNotToPerformAssertions(); + + return; + } + + // Short of __FUNCTION__: the engine caps a table name at 64 bytes and the suite's + // per-run prefix leaves this method's name no room. + $collectionId = 'metadataSchemaUpdate'; + + $database->createCollection(Collection::create(id: $collectionId)); + $this->assertSame(false, $database->isDroppingUnknownAttributes()); + + // What a patch script ahead of the release writes: the physical column alone. The + // metadata schema is declared in code, so there is no stored definition to keep in step. + $adapter->createAttribute( + Database::METADATA, + Attribute::string(key: 'unreleased', size: 1_000_000, filters: [Filter::Json]), + ); + + try { + // Drop the cached definition, so the next read is the one that brings the column back. + $database->purgeCachedDocument(Database::METADATA, $collectionId); + + // The read hands the column straight back, undeclared and all, because the adapter + // selects every column. This is the document the schema operation writes back, so + // without it arriving here there would be nothing for the exemption to strip. + $this->assertTrue( + $database->getCollection($collectionId)->offsetExists('unreleased'), + 'The undeclared column never reached the definition, so the case under test did not arise' + ); + + $this->assertSame( + 'known', + $database->createAttribute($collectionId, Attribute::string(key: 'known', size: 128))->key, + 'A schema operation failed against a metadata table holding an undeclared column' + ); + + // The caller's own table is untouched by this: an attribute it does not declare is + // still refused rather than discarded. + try { + $database->createDocument($collectionId, new Document([ + '$id' => 'strict', + '$permissions' => [Permission::read(Role::any())], + 'known' => 'kept', + 'unknown' => 'refused', + ])); + $this->fail('Unknown attribute was accepted on a caller-owned table'); + } catch (StructureException $e) { + $this->assertEquals('Invalid document structure: Unknown attribute: "unknown"', $e->getMessage()); + } + } finally { + $adapter->deleteAttribute(Database::METADATA, 'unreleased'); + $database->purgeCachedDocument(Database::METADATA, $collectionId); + } + + $database->deleteCollection($collectionId); + } + public function testCreateDocumentWithBigIntType(): void { $database = $this->getDatabase(); From b18c4f697fe23cd2048b9f617d920550a94c3757 Mon Sep 17 00:00:00 2001 From: fogelito Date: Tue, 17 Nov 2026 12:21:00 +0200 Subject: [PATCH 5/5] fix --- tests/e2e/Adapter/Scopes/DocumentTests.php | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index 87857479d..a7b29b9a5 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -9904,10 +9904,12 @@ public function testMetadataDropsUnknownAttributes(): void * that declares it, and a schema operation then has to survive reading it back. Without the * exemption the definition write fails structure validation and rolls its own DDL back. * - * SQLite is left out because only the cleanup is unportable: deleteAttribute rebuilds a - * table from the stored definition row that _metadata has none of, so the column could not - * be taken off again, and it is shared with every test after this one. The flag-independent - * branch itself is covered on every adapter by testMetadataDropsUnknownAttributes. + * Only an engine with a fixed physical schema can hold the column this is about: one that + * exists on the table while no document carries it. Mongo has no such notion, so there the + * case cannot be staged at all. SQLite could stage it but not clear it up, because + * deleteAttribute rebuilds a table from the stored definition row that _metadata has none + * of, and _metadata is shared with every test after this one. The flag-independent branch + * itself is covered on every adapter by testMetadataDropsUnknownAttributes. */ public function testMetadataDropsUnknownAttributesOnSchemaUpdate(): void { @@ -9921,7 +9923,12 @@ public function testMetadataDropsUnknownAttributesOnSchemaUpdate(): void return; } - if ($adapter instanceof SQLite || $adapter instanceof Memory || $adapter instanceof Redis) { + if ( + $adapter instanceof SQLite + || $adapter instanceof Memory + || $adapter instanceof Redis + || $adapter instanceof Mongo + ) { $this->expectNotToPerformAssertions(); return;