From 3a88371df54d3b25794bac9edaff4b01e716c2ce Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 20:19:12 -0800 Subject: [PATCH 1/3] fix(db): bootstrap Projects for development push databases that predate them A push-managed database with workspaces but no Project schema could never pass the schema-push guard: push cannot backfill workspace.project_id, and the guard refuses every unenforced Project state. Under SIM_DEV_DB_PUSH=1 the pre-push reconciler now replays migrations 0394 and 0406 (the foundation commits atomically with the rollout marker) and runs the registered 0032_project_membership script migration on its single session, recording it as the migration runner does. An interrupted bootstrap resumes; every other unenforced state still refuses with the same message. The pinned drizzle-kit patch also adds unique constraints before foreign keys, so a composite foreign key on a new unique constraint of an existing table no longer fails, and in the noninteractive push policy it answers the unique-constraint truncate prompt with its non-destructive choice. --- .../project-push-bootstrap.integration.ts | 340 ++++++++++++++++++ packages/db/scripts/push.integration.ts | 19 + .../scripts/reconcile-project-membership.ts | 101 +++++- patches/README.md | 12 +- patches/drizzle-kit@0.31.10.patch | 15 + 5 files changed, 478 insertions(+), 9 deletions(-) create mode 100644 packages/db/scripts/project-push-bootstrap.integration.ts diff --git a/packages/db/scripts/project-push-bootstrap.integration.ts b/packages/db/scripts/project-push-bootstrap.integration.ts new file mode 100644 index 00000000000..3779e227ebb --- /dev/null +++ b/packages/db/scripts/project-push-bootstrap.integration.ts @@ -0,0 +1,340 @@ +import { spawnSync } from 'node:child_process' +import { readFile } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import journal from '@sim/db/migrations/meta/_journal.json' +import { applyMigration } from '@sim/db/scripts/migration-fixture' +import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' +import { generateId } from '@sim/utils/id' +import postgres, { type Sql } from 'postgres' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +const REFUSAL = + 'Existing Project databases must complete the Project script migration 0032_project_membership before schema push' +const MIGRATION_LOCK_KEY = 4_961_002_270n +const packageRoot = fileURLToPath(new URL('..', import.meta.url)) +const databaseUrl = readTestDatabaseUrl() + +function databaseUrlFor(name: string): string { + const url = new URL(databaseUrl) + url.pathname = `/${name}` + return url.toString() +} + +/** Runs the supported `db:push --force` entrypoint, fence included, against one database. */ +function runPush(url: string, devPush: boolean) { + const result = spawnSync('bun', ['--no-env-file', 'scripts/push.ts', '--force'], { + cwd: packageRoot, + env: { + ...process.env, + DATABASE_URL: url, + MIGRATION_DATABASE_URL: url, + /** The test runner's NODE_ENV silences the child loggers whose messages these tests read. */ + NODE_ENV: 'development', + SIM_DEV_DB_PUSH: devPush ? '1' : undefined, + }, + encoding: 'utf8', + timeout: 120_000, + }) + expect(result.error).toBeUndefined() + return { status: result.status, output: result.stdout + result.stderr } +} + +async function migrationSql(tag: string): Promise { + return readFile(new URL(`../migrations/${tag}.sql`, import.meta.url), 'utf8') +} + +/** Replays a versioned migration on one session, as the migrator does. */ +async function replayMigration(sql: Sql, tag: string): Promise { + const session = await sql.reserve() + try { + await applyMigration(session, await migrationSql(tag)) + } finally { + session.release() + } +} + +async function projectSchema(sql: Sql) { + const [state] = await sql`SELECT + to_regclass('public.project') IS NOT NULL AS project, + to_regclass('public.project_workspace') IS NOT NULL AS connector, + to_regclass('public.project_membership_rollout') IS NOT NULL AS rollout, + (SELECT attnotnull FROM pg_attribute WHERE attrelid = 'public.workspace'::regclass + AND attname = 'project_id' AND NOT attisdropped) AS required` + return state +} + +/** Everything a refused push must leave exactly as it found it. */ +async function retainedState(sql: Sql) { + const schema = await projectSchema(sql) + return { + schema, + workspaces: await sql`SELECT to_jsonb(w) AS row FROM workspace w ORDER BY id`, + connector: schema.connector + ? await sql`SELECT * FROM project_workspace ORDER BY workspace_id` + : [], + } +} + +describe('development schema push of a database that predates Projects', () => { + const template = `project_push_template_test_${generateId().replaceAll('-', '')}` + let admin: Sql + + beforeAll(async () => { + admin = postgres(databaseUrl, { max: 1, onnotice: () => undefined }) + await admin.unsafe(`CREATE DATABASE "${template}"`) + const sql = postgres(databaseUrlFor(template), { max: 1, onnotice: () => undefined }) + try { + for (const extension of ['vector', 'btree_gin', 'pg_trgm']) { + await sql`CREATE EXTENSION IF NOT EXISTS ${sql(extension)}` + } + const fresh = runPush(databaseUrlFor(template), true) + expect(fresh.status, fresh.output).toBe(0) + /** Removes every Project object so the template matches a push-managed schema from before Projects. */ + await sql.unsafe(` + DROP TABLE project CASCADE; + ALTER TABLE workspace DROP COLUMN project_id CASCADE, DROP COLUMN organization_scope_key CASCADE; + ALTER TABLE folder DROP COLUMN project_id CASCADE; + ALTER TABLE workspace_files DROP COLUMN project_id CASCADE; + DELETE FROM script_migrations WHERE name = '0032_project_membership'; + `) + } finally { + await sql.end() + } + }, 180_000) + + afterAll(async () => { + if (!admin) return + await admin.unsafe(`DROP DATABASE IF EXISTS "${template}" WITH (FORCE)`) + await admin.end() + }) + + /** Copies the pre-Project template with seeded tenants into a database owned by one test. */ + async function preProjectDatabase(run: (sql: Sql, url: string) => Promise) { + const name = `project_push_test_${generateId().replaceAll('-', '')}` + await admin.unsafe(`CREATE DATABASE "${name}" TEMPLATE "${template}"`) + const url = databaseUrlFor(name) + const sql = postgres(url, { max: 2, onnotice: () => undefined }) + try { + await sql.unsafe(` + INSERT INTO "user" (id, name, email, email_verified, created_at, updated_at) VALUES + ('owner', 'Owner', 'owner@example.test', true, now(), now()), + ('member', 'Member', 'member@example.test', true, now(), now()); + INSERT INTO organization (id, name, slug) VALUES ('org', 'Org', 'org'); + INSERT INTO workspace (id, name, owner_id, billed_account_user_id, organization_id, forked_from_workspace_id, archived_at) VALUES + ('personal', 'Personal', 'owner', 'owner', NULL, NULL, NULL), + ('team', 'Team', 'owner', 'owner', 'org', NULL, NULL), + ('team-fork', 'Team fork', 'member', 'owner', 'org', 'team', NULL), + ('retired', 'Retired', 'member', 'member', NULL, NULL, '2026-01-01'); + INSERT INTO folder (id, resource_type, name, user_id, workspace_id) VALUES + ('folder', 'workflow', 'Folder', 'owner', 'team'); + `) + await run(sql, url) + } finally { + await sql.end({ timeout: 2 }) + await admin.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`) + } + } + + async function expectBootstrapped(sql: Sql) { + expect(await projectSchema(sql)).toEqual({ + project: true, + connector: false, + rollout: false, + required: true, + }) + const memberships = await sql<{ id: string; project_id: string; archived: boolean }[]>` + SELECT w.id, w.project_id, p.archived_at IS NOT NULL AS archived + FROM workspace w JOIN project p ON p.id = w.project_id ORDER BY w.id` + expect(memberships.map(({ id, archived }) => ({ id, archived }))).toEqual([ + { id: 'personal', archived: false }, + { id: 'retired', archived: true }, + { id: 'team', archived: false }, + { id: 'team-fork', archived: false }, + ]) + const projectOf = new Map(memberships.map((row) => [row.id, row.project_id])) + expect(projectOf.get('team-fork')).toBe(projectOf.get('team')) + expect(new Set(projectOf.values()).size).toBe(3) + expect( + await sql`SELECT conname, convalidated, condeferrable FROM pg_constraint + WHERE conrelid = 'workspace'::regclass AND conname IN ('workspace_project_id_project_id_fk', + 'workspace_project_organization_fk', 'workspace_fork_project_fk') + ORDER BY conname` + ).toEqual([ + { conname: 'workspace_fork_project_fk', convalidated: true, condeferrable: true }, + { conname: 'workspace_project_id_project_id_fk', convalidated: true, condeferrable: false }, + { conname: 'workspace_project_organization_fk', convalidated: true, condeferrable: true }, + ]) + expect( + await sql`SELECT name FROM script_migrations WHERE name = '0032_project_membership'` + ).toEqual([{ name: '0032_project_membership' }]) + await expect( + sql`UPDATE workspace SET organization_id = NULL WHERE id = 'team-fork'` + ).rejects.toMatchObject({ code: '23503' }) + } + + it('bootstraps Projects, completes the push, and keeps memberships and constraints on replay', async () => { + await preProjectDatabase(async (sql, url) => { + const pushed = runPush(url, true) + expect(pushed.status, pushed.output).toBe(0) + await expectBootstrapped(sql) + const before = await sql`SELECT id, project_id FROM workspace ORDER BY id` + const replayed = runPush(url, true) + expect(replayed.status, replayed.output).toBe(0) + expect(await sql`SELECT id, project_id FROM workspace ORDER BY id`).toEqual(before) + await expectBootstrapped(sql) + }) + }, 240_000) + + it('resumes after the Project migration reports conflicts', async () => { + await preProjectDatabase(async (sql, url) => { + await sql`INSERT INTO workspace (id, name, owner_id, billed_account_user_id, forked_from_workspace_id) + VALUES ('cycle-a', 'Cycle A', 'owner', 'owner', NULL), ('cycle-b', 'Cycle B', 'owner', 'owner', 'cycle-a')` + await sql`UPDATE workspace SET forked_from_workspace_id = 'cycle-b' WHERE id = 'cycle-a'` + const refused = runPush(url, true) + expect(refused.status, refused.output).toBe(1) + expect(refused.output).toContain('Project preparation requires operator remediation') + expect(await projectSchema(sql)).toEqual({ + project: true, + connector: true, + rollout: true, + required: false, + }) + expect(await sql`SELECT phase FROM project_membership_rollout`).toEqual([{ phase: 'column' }]) + expect( + await sql`SELECT name FROM script_migrations WHERE name = '0032_project_membership'` + ).toEqual([]) + + await sql`UPDATE workspace SET forked_from_workspace_id = NULL WHERE id = 'cycle-a'` + const resumed = runPush(url, true) + expect(resumed.status, resumed.output).toBe(0) + await sql`DELETE FROM workspace WHERE id IN ('cycle-a', 'cycle-b')` + await expectBootstrapped(sql) + }) + }, 240_000) + + it('rolls the foundation back with the expansion when the workspace barrier is busy', async () => { + await preProjectDatabase(async (sql, url) => { + const holder = await sql.reserve() + try { + await holder`BEGIN` + await holder`SELECT 1 FROM workspace LIMIT 1` + const busy = runPush(url, true) + expect(busy.status, busy.output).toBe(1) + expect(busy.output).toContain('55P03') + await holder`ROLLBACK` + } finally { + holder.release() + } + expect(await projectSchema(sql)).toEqual({ + project: false, + connector: false, + rollout: false, + required: null, + }) + const pushed = runPush(url, true) + expect(pushed.status, pushed.output).toBe(0) + await expectBootstrapped(sql) + }) + }, 240_000) + + it('leaves the database untouched outside development push mode', async () => { + await preProjectDatabase(async (sql, url) => { + const refused = runPush(url, false) + expect(refused.status, refused.output).toBe(1) + expect(refused.output).toContain(REFUSAL) + expect(await projectSchema(sql)).toEqual({ + project: false, + connector: false, + rollout: false, + required: null, + }) + expect( + await sql`SELECT name FROM script_migrations WHERE name = '0032_project_membership'` + ).toEqual([]) + }) + }, 120_000) + + it('refuses to bootstrap while a migration holds the schema lock', async () => { + await preProjectDatabase(async (sql, url) => { + const migrator = await sql.reserve() + try { + await migrator`SELECT pg_advisory_lock(${MIGRATION_LOCK_KEY})` + const fenced = runPush(url, true) + expect(fenced.status, fenced.output).toBe(1) + expect(fenced.output).toContain('A migration is running') + await migrator`SELECT pg_advisory_unlock(${MIGRATION_LOCK_KEY})` + } finally { + migrator.release() + } + expect((await projectSchema(sql)).project).toBe(false) + }) + }, 120_000) + + it.each([ + { + state: 'a connector with live memberships', + async build(sql: Sql) { + await sql.unsafe(await migrationSql('0394_project_foundation')) + await sql`INSERT INTO project (id, name, owner_id) VALUES ('legacy', 'Legacy', 'owner')` + await sql`INSERT INTO project_workspace (project_id, workspace_id) VALUES ('legacy', 'personal')` + }, + }, + { + state: 'an empty connector without the column expansion', + async build(sql: Sql) { + await sql.unsafe(await migrationSql('0394_project_foundation')) + }, + }, + { + state: 'Projects without a connector or required column', + async build(sql: Sql) { + await sql.unsafe(await migrationSql('0394_project_foundation')) + await sql`DROP TABLE project_workspace` + }, + }, + { + state: 'a workspace Project column without Projects', + async build(sql: Sql) { + await sql`ALTER TABLE workspace ADD COLUMN project_id text` + }, + }, + { + state: 'a versioned expansion awaiting its script migration', + async build(sql: Sql) { + await sql.unsafe(await migrationSql('0394_project_foundation')) + await replayMigration(sql, '0406_workspace_project_column') + const expansion = journal.entries.find( + (entry) => entry.tag === '0406_workspace_project_column' + ) + await sql`CREATE SCHEMA drizzle` + await sql`CREATE TABLE drizzle.__drizzle_migrations (id serial PRIMARY KEY, hash text NOT NULL, created_at bigint)` + await sql`INSERT INTO drizzle.__drizzle_migrations (hash, created_at) VALUES ('fixture', ${expansion?.when ?? 0})` + }, + }, + { + state: 'an expansion whose connector holds memberships', + async build(sql: Sql) { + await sql.unsafe(await migrationSql('0394_project_foundation')) + await replayMigration(sql, '0406_workspace_project_column') + await sql`INSERT INTO project (id, name, owner_id) VALUES ('legacy', 'Legacy', 'owner')` + await sql`INSERT INTO project_workspace (project_id, workspace_id) VALUES ('legacy', 'personal')` + }, + }, + ])( + 'refuses development push of $state', + async ({ build }) => { + await preProjectDatabase(async (sql, url) => { + await build(sql) + const before = await retainedState(sql) + const refused = runPush(url, true) + expect(refused.status, refused.output).toBe(1) + expect(refused.output).toContain(REFUSAL) + expect(await retainedState(sql)).toEqual(before) + expect( + await sql`SELECT name FROM script_migrations WHERE name = '0032_project_membership'` + ).toEqual([]) + }) + }, + 120_000 + ) +}) diff --git a/packages/db/scripts/push.integration.ts b/packages/db/scripts/push.integration.ts index 576e1afc878..dd20a82a87e 100644 --- a/packages/db/scripts/push.integration.ts +++ b/packages/db/scripts/push.integration.ts @@ -330,6 +330,25 @@ export const workflows = pgTable('workflow', { ).rejects.toMatchObject({ code: '23503' }) }, 60_000) + it('adds a composite unique constraint before the foreign keys that reference it', async () => { + await sql`CREATE TABLE folder (id text PRIMARY KEY, resource_type text NOT NULL, parent_id text)` + await sql`INSERT INTO folder VALUES ('parent', 'file', NULL), ('child', 'file', 'parent')` + await schema(`export const folders = pgTable('folder', { + id: text('id').primaryKey(), resourceType: text('resource_type').notNull(), + projectId: text('project_id'), parentId: text('parent_id'), +}, (t) => [ + unique('folder_project_owner_type_unique').on(t.id, t.projectId, t.resourceType), + foreignKey({ name: 'folder_project_parent_owner_fk', columns: [t.parentId, t.projectId, t.resourceType], foreignColumns: [t.id, t.projectId, t.resourceType] }), +])`) + const result = push() + expect(result.error).toBeUndefined() + expect(result.status, result.stdout + result.stderr).toBe(0) + await sql`UPDATE folder SET project_id = 'project' WHERE id = 'parent'` + await expect( + sql`UPDATE folder SET project_id = 'other' WHERE id = 'child'` + ).rejects.toMatchObject({ code: '23503' }) + }, 30_000) + it('retires the legacy size bridge without losing bigint or unbackfilled values', async () => { await sql`CREATE TABLE workspace_files (id text PRIMARY KEY, size integer NOT NULL, size_bytes bigint)` await sql`INSERT INTO workspace_files VALUES ('legacy', 123, NULL), ('large', 2147483647, 5000000000)` diff --git a/packages/db/scripts/reconcile-project-membership.ts b/packages/db/scripts/reconcile-project-membership.ts index cea52f5f292..29cc4b0a3ea 100644 --- a/packages/db/scripts/reconcile-project-membership.ts +++ b/packages/db/scripts/reconcile-project-membership.ts @@ -1,4 +1,8 @@ +import { readFile } from 'node:fs/promises' import { enforceProjectMembership } from '@sim/db/maintenance/project-enforcement' +import journal from '@sim/db/migrations/meta/_journal.json' +import { runScriptMigrations } from '@sim/db/script-migrations' +import { projectMembershipMigration } from '@sim/db/script-migrations/0032_project_membership' import { createLogger } from '@sim/logger' import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import postgres from 'postgres' @@ -7,22 +11,103 @@ const logger = createLogger('ProjectMembershipPush') const url = process.env.MIGRATION_DATABASE_URL || process.env.DATABASE_URL if (!url) throw new Error('A database URL is required for Project schema push reconciliation') const sql = postgres(url, { max: 1, prepare: false, connect_timeout: 10, max_lifetime: null }) -try { - const [state] = await sql< - { workspace: boolean; project: boolean; connector: boolean; required: boolean }[] - >` + +interface ProjectSchemaState { + workspace: boolean + project: boolean + connector: boolean + rollout: boolean + journal: boolean + /** Null when `workspace.project_id` does not exist; otherwise whether it is NOT NULL. */ + projectIdNotNull: boolean | null +} + +async function readProjectSchemaState(): Promise { + const [state] = await sql` SELECT to_regclass('public.workspace') IS NOT NULL AS workspace, to_regclass('public.project') IS NOT NULL AS project, to_regclass('public.project_workspace') IS NOT NULL AS connector, - EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = to_regclass('public.workspace') - AND attname = 'project_id' AND NOT attisdropped AND attnotnull) AS required + to_regclass('public.project_membership_rollout') IS NOT NULL AS rollout, + to_regclass('drizzle.__drizzle_migrations') IS NOT NULL AS journal, + (SELECT attnotnull FROM pg_attribute WHERE attrelid = to_regclass('public.workspace') + AND attname = 'project_id' AND NOT attisdropped) AS "projectIdNotNull" ` - if (state.workspace && (state.connector || !state.required || !state.project)) { + return state +} + +/** + * Classifies a development push database that has no connector writer to drain: either it + * predates Projects entirely, or an earlier bootstrap committed the expansion and stopped before + * enforcement. Only the versioned 0406 creates the rollout marker, so a marker that the migration + * journal does not account for, beside a connector nothing has written, is a bootstrap to resume. + * Every other shape returns null and keeps the writer-drain refusal. + */ +async function classifyProjectBootstrap( + state: ProjectSchemaState +): Promise<'fresh' | 'resume' | null> { + if (!state.workspace || state.projectIdNotNull) return null + const parts = [state.project, state.connector, state.rollout, state.projectIdNotNull !== null] + if (parts.every((present) => !present)) return 'fresh' + if (!parts.every(Boolean)) return null + const [connector] = await sql<{ populated: boolean }[]>` + SELECT EXISTS (SELECT 1 FROM public.project_workspace) AS populated` + if (connector.populated) return null + if (state.journal) { + const expansion = journal.entries.find((entry) => entry.tag === '0406_workspace_project_column') + if (!expansion) + throw new Error('Project column expansion is missing from the migration journal') + const [applied] = await sql<{ versioned: boolean }[]>` + SELECT EXISTS (SELECT 1 FROM drizzle.__drizzle_migrations + WHERE created_at = ${expansion.when}) AS versioned` + if (applied.versioned) return null + } + return 'resume' +} + +async function migrationStatements(tag: string): Promise { + const source = await readFile(new URL(`../migrations/${tag}.sql`, import.meta.url), 'utf8') + return source.split('--> statement-breakpoint').filter((statement) => statement.trim()) +} + +/** + * Applies the versioned Project expansion, then the registered script migration on this + * single-session connection, recording it as the migration runner does. 0406 opens by committing + * the migrator's batch; leaving that out commits the 0394 foundation together with the rollout + * marker, so an interrupted bootstrap leaves either no Project schema or one it can resume. + */ +async function bootstrapProjects(mode: 'fresh' | 'resume'): Promise { + logger.info('Bootstrapping Projects for a development push database without them', { mode }) + const foundation = mode === 'fresh' ? await migrationStatements('0394_project_foundation') : [] + const [batchCommit, ...expansion] = await migrationStatements('0406_workspace_project_column') + if (batchCommit.trim() !== 'COMMIT;') { + throw new Error('Project column expansion no longer opens with the migrator batch commit') + } + await sql.unsafe('BEGIN') + try { + for (const statement of [...foundation, ...expansion]) await sql.unsafe(statement) + } catch (error) { + await sql.unsafe('ROLLBACK').catch(() => undefined) + throw error + } + await runScriptMigrations(sql, [projectMembershipMigration]) +} + +const prepare = process.argv.includes('--prepare') +try { + let state = await readProjectSchemaState() + if (prepare && process.env.SIM_DEV_DB_PUSH === '1') { + const bootstrap = await classifyProjectBootstrap(state) + if (bootstrap) { + await bootstrapProjects(bootstrap) + state = await readProjectSchemaState() + } + } + if (state.workspace && (state.connector || !state.projectIdNotNull || !state.project)) { throw new Error( 'Existing Project databases must complete the Project script migration 0032_project_membership before schema push. Use the phased migration path for retained data; schema push cannot bypass the writer-drain requirement' ) } - if (!process.argv.includes('--prepare') && state.workspace && state.project) { + if (!prepare && state.workspace && state.project) { /** Drizzle cannot express deferred FK timing; fresh push shares the migration finalizer. */ await enforceProjectMembership(sql) logger.info('Project membership validation and native constraints completed') diff --git a/patches/README.md b/patches/README.md index c5bc00eff79..b97ad405fda 100644 --- a/patches/README.md +++ b/patches/README.md @@ -60,11 +60,21 @@ chooser and requires a terminal. Normal migration generation keeps its rename prompts. Data-loss confirmations, table filters, introspection, and generated SQL remain owned by Drizzle; `--force` still controls data-loss approval. +The same policy answers Drizzle's prompt to truncate a populated table before +adding a unique constraint with its non-destructive choice: the constraint is +added without truncating, and duplicate rows fail the push. Drizzle also adds +PostgreSQL unique constraints after the foreign keys of the same diff, so a +composite foreign key that references a new unique constraint on an existing +table (the folder ownership keys) always fails. The patch adds unique +constraints right after new columns, before every foreign key, for push and +generated migrations alike; a unique constraint never depends on a foreign key. + `packages/db/scripts/push.test.ts` covers argument forwarding and stopping before reconciliation on failure. `packages/db/scripts/push.integration.ts` exercises the installed CLI against PostgreSQL, including multiple column changes, table and enum replacement, schema replacement, preservation of the excluded script -ledger, and database errors. Set `TEST_DATABASE_URL` to a disposable local +ledger, database errors, and a composite foreign key on a new unique constraint of +a populated table. Set `TEST_DATABASE_URL` to a disposable local PostgreSQL database and run these tests with `bun run --cwd packages/db test --mode integration`. diff --git a/patches/drizzle-kit@0.31.10.patch b/patches/drizzle-kit@0.31.10.patch index 67c42f08929..841c851274b 100644 --- a/patches/drizzle-kit@0.31.10.patch +++ b/patches/drizzle-kit@0.31.10.patch @@ -13,6 +13,16 @@ diff --git a/bin.cjs b/bin.cjs + WHERE tc.table_name = '${tableName}' and constraint_schema = '${tableSchema}' + ORDER BY constraint_name, kcu.ordinal_position;` ); +@@ -29040,8 +29041,8 @@ + jsonStatements.push(...jsonAddColumnsStatemets); ++ jsonStatements.push(...jsonAddedUniqueConstraints); + jsonStatements.push(...jsonCreateReferencesForCreatedTables); + jsonStatements.push(...jsonCreateIndexesForCreatedTables); + jsonStatements.push(...jsonCreatedReferencesForAlteredTables); + jsonStatements.push(...jsonCreateIndexesFoAlteredTables); + jsonStatements.push(...jsonDropColumnsStatemets); + jsonStatements.push(...jsonAlteredCompositePKs); +- jsonStatements.push(...jsonAddedUniqueConstraints); @@ -32694,3 +32695,3 @@ promptColumnsConflicts = async (tableName, newColumns, missingColumns) => { - if (newColumns.length === 0 || missingColumns.length === 0) { @@ -33,6 +43,11 @@ diff --git a/bin.cjs b/bin.cjs - if (missingSchemas.length === 0 || newSchemas.length === 0) { + if (process.env.SIM_DB_PUSH_RENAME_MODE === "create" || missingSchemas.length === 0 || newSchemas.length === 0) { return { created: newSchemas, renamed: [], deleted: missingSchemas }; +@@ -80441,3 +80442,3 @@ + const count = Number(res[0].count); +- if (count > 0) { ++ if (count > 0 && process.env.SIM_DB_PUSH_RENAME_MODE !== "create") { + const unsquashedUnique = PgSquasher.unsquashUnique(statement.data); @@ -82741,3 +82742,3 @@ (0, import_hanji13.render)(`[${source_default.red("x")}] All changes were aborted`); - process.exit(0); From fbbd6df08956505fe7325599f61d3e28c9b99668 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 20:42:50 -0800 Subject: [PATCH 2/3] fix(db): push the schema before the Project migration reads it The bootstrap ran 0032_project_membership before Drizzle pushed anything, so a development database from before a column the migration reads (such as the fork parent) failed after committing the expansion, on every retry. --prepare now only replays the expansion. push.ts then runs one Drizzle push with workspace.project_id still nullable (SIM_DB_PUSH_PROJECT_BOOTSTRAP=1), runs the migration through reconcile-project-membership.ts --bootstrap, and finishes with the ordinary push. The connector and rollout marker are kept out of Drizzle's diff until the migration retires them. A fresh bootstrap also requires that no public table carries a project_id column. --- packages/db/drizzle.config.ts | 7 +- packages/db/schema.ts | 18 ++++- .../project-push-bootstrap.integration.ts | 31 ++++++++ packages/db/scripts/push.ts | 51 +++++++++--- .../scripts/reconcile-project-membership.ts | 79 ++++++++++++------- 5 files changed, 141 insertions(+), 45 deletions(-) diff --git a/packages/db/drizzle.config.ts b/packages/db/drizzle.config.ts index 23aef61ba21..ead9081ec44 100644 --- a/packages/db/drizzle.config.ts +++ b/packages/db/drizzle.config.ts @@ -15,9 +15,14 @@ export default { dbCredentials: { url: process.env.DATABASE_URL!, }, - /** Runner-owned journals and resumable cleanup cursors must survive a development schema push. */ + /** + * Runner-owned journals, resumable cleanup cursors, and a Project bootstrap's connector and + * rollout marker (until its migration retires them) must survive a development schema push. + */ tablesFilter: [ '!script_migrations', + '!project_workspace', + '!project_membership_rollout', '!project_backfill_archive_repairs', '!search_embedding_cleanup_progress', '!search_embedding_cleanup_targets', diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 0ebce86d034..7a78be87da7 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -2171,6 +2171,20 @@ export const workspaceModeEnum = pgEnum('workspace_mode', [ export type WorkspaceMode = (typeof workspaceModeEnum.enumValues)[number] +/** + * A development push that bootstraps Projects first pushes this column nullable, so the columns + * `0032_project_membership` reads exist before it fills the column and enforces NOT NULL; the + * push that follows uses the required column. Every other push and generation reads it required. + */ +function workspaceProjectIdColumn() { + const column = text('project_id').references((): AnyPgColumn => project.id, { + onDelete: 'restrict', + }) + return process.env.SIM_DB_PUSH_PROJECT_BOOTSTRAP === '1' + ? (column as ReturnType) + : column.notNull() +} + export const workspace = pgTable( 'workspace', { @@ -2198,9 +2212,7 @@ export const workspace = pgTable( organizationScopeKey: text('organization_scope_key').generatedAlwaysAs( sql`CASE WHEN organization_id IS NULL THEN 'personal' ELSE 'organization:' || organization_id END` ), - projectId: text('project_id') - .notNull() - .references((): AnyPgColumn => project.id, { onDelete: 'restrict' }), + projectId: workspaceProjectIdColumn(), workspaceMode: workspaceModeEnum('workspace_mode').notNull().default('grandfathered_shared'), billedAccountUserId: text('billed_account_user_id') .notNull() diff --git a/packages/db/scripts/project-push-bootstrap.integration.ts b/packages/db/scripts/project-push-bootstrap.integration.ts index 3779e227ebb..32a2bffa634 100644 --- a/packages/db/scripts/project-push-bootstrap.integration.ts +++ b/packages/db/scripts/project-push-bootstrap.integration.ts @@ -185,6 +185,30 @@ describe('development schema push of a database that predates Projects', () => { }) }, 240_000) + it('pushes the columns the Project migration reads before running it', async () => { + await preProjectDatabase(async (sql, url) => { + await sql`ALTER TABLE workspace DROP COLUMN forked_from_workspace_id CASCADE` + const pushed = runPush(url, true) + expect(pushed.status, pushed.output).toBe(0) + expect(await projectSchema(sql)).toEqual({ + project: true, + connector: false, + rollout: false, + required: true, + }) + expect( + await sql`SELECT count(*)::int AS unassigned FROM workspace WHERE project_id IS NULL` + ).toEqual([{ unassigned: 0 }]) + expect( + await sql`SELECT count(*)::int AS restored FROM pg_attribute WHERE attrelid = 'workspace'::regclass + AND attname = 'forked_from_workspace_id' AND NOT attisdropped` + ).toEqual([{ restored: 1 }]) + expect( + await sql`SELECT name FROM script_migrations WHERE name = '0032_project_membership'` + ).toEqual([{ name: '0032_project_membership' }]) + }) + }, 240_000) + it('resumes after the Project migration reports conflicts', async () => { await preProjectDatabase(async (sql, url) => { await sql`INSERT INTO workspace (id, name, owner_id, billed_account_user_id, forked_from_workspace_id) @@ -298,6 +322,13 @@ describe('development schema push of a database that predates Projects', () => { await sql`ALTER TABLE workspace ADD COLUMN project_id text` }, }, + { + state: 'folder Project ownership without Projects', + async build(sql: Sql) { + await sql`ALTER TABLE folder ADD COLUMN project_id text` + await sql`UPDATE folder SET project_id = 'orphaned'` + }, + }, { state: 'a versioned expansion awaiting its script migration', async build(sql: Sql) { diff --git a/packages/db/scripts/push.ts b/packages/db/scripts/push.ts index e0417ce76c1..89a0492c089 100644 --- a/packages/db/scripts/push.ts +++ b/packages/db/scripts/push.ts @@ -17,8 +17,13 @@ const RECONCILIATION_COMMANDS = [ ['bun', '--env-file=.env', 'run', './script-migrations/0031_drop_keyword_tin.ts'], ] +/** Reads, on the fence session, whether `--prepare` left a Project bootstrap for this push to finish. */ +type ProjectBootstrapProbe = () => Promise + /** Fences preparation, Drizzle and reconcilers from a concurrent migration run. */ -async function withMigrationFence(run: (signal: AbortSignal) => Promise): Promise { +async function withMigrationFence( + run: (signal: AbortSignal, projectBootstrapPending: ProjectBootstrapProbe) => Promise +): Promise { const rawUrl = process.env.DATABASE_URL if (!rawUrl) { logger.error('DATABASE_URL is required for schema push') @@ -48,7 +53,11 @@ async function withMigrationFence(run: (signal: AbortSignal) => Promise) logger.error('A migration is running; retry schema push later') return 1 } - return run(abort.signal) + return run(abort.signal, async () => { + const [state] = await tx<{ pending: boolean }[]>` + SELECT to_regclass('public.project_membership_rollout') IS NOT NULL AS pending`.simple() + return state.pending + }) })) as number } catch (error) { logger.error( @@ -82,11 +91,21 @@ export async function runPush(args: string[]): Promise { return 1 } - async function runCommands(signal?: AbortSignal): Promise { - async function runCommand(command: string[]): Promise { + async function runCommands( + signal?: AbortSignal, + projectBootstrapPending?: ProjectBootstrapProbe + ): Promise { + async function runCommand( + command: string[], + env: Record = {} + ): Promise { signal?.throwIfAborted() const child = Bun.spawn(command, { - env: { ...process.env, SIM_DB_PUSH_RENAME_MODE: interactiveRenames ? 'prompt' : 'create' }, + env: { + ...process.env, + SIM_DB_PUSH_RENAME_MODE: interactiveRenames ? 'prompt' : 'create', + ...env, + }, stdin: 'inherit', stdout: 'inherit', stderr: 'inherit', @@ -119,13 +138,21 @@ export async function runPush(args: string[]): Promise { const pushArgs = args.filter((arg) => arg !== '--interactive-renames') // Launch the installed CLI directly so losing the fence also terminates its database connection. const cli = fileURLToPath(new URL('bin.cjs', import.meta.resolve('drizzle-kit'))) - const exitCode = await runCommand([ - 'bun', - cli, - 'push', - '--config=./drizzle.config.ts', - ...pushArgs, - ]) + const drizzlePush = ['bun', cli, 'push', '--config=./drizzle.config.ts', ...pushArgs] + if (!help && process.env.SIM_DEV_DB_PUSH === '1' && (await projectBootstrapPending?.())) { + /** The Project migration reads columns an older database gains only from this push. */ + const code = await runCommand(drizzlePush, { SIM_DB_PUSH_PROJECT_BOOTSTRAP: '1' }) + if (code !== 0) return code + const bootstrapCode = await runCommand([ + 'bun', + '--env-file=.env', + 'run', + './scripts/reconcile-project-membership.ts', + '--bootstrap', + ]) + if (bootstrapCode !== 0) return bootstrapCode + } + const exitCode = await runCommand(drizzlePush) if (exitCode !== 0 || help) return exitCode for (const command of RECONCILIATION_COMMANDS) { diff --git a/packages/db/scripts/reconcile-project-membership.ts b/packages/db/scripts/reconcile-project-membership.ts index 29cc4b0a3ea..0495ba4e031 100644 --- a/packages/db/scripts/reconcile-project-membership.ts +++ b/packages/db/scripts/reconcile-project-membership.ts @@ -12,6 +12,9 @@ const url = process.env.MIGRATION_DATABASE_URL || process.env.DATABASE_URL if (!url) throw new Error('A database URL is required for Project schema push reconciliation') const sql = postgres(url, { max: 1, prepare: false, connect_timeout: 10, max_lifetime: null }) +const REFUSAL = + 'Existing Project databases must complete the Project script migration 0032_project_membership before schema push. Use the phased migration path for retained data; schema push cannot bypass the writer-drain requirement' + interface ProjectSchemaState { workspace: boolean project: boolean @@ -20,6 +23,8 @@ interface ProjectSchemaState { journal: boolean /** Null when `workspace.project_id` does not exist; otherwise whether it is NOT NULL. */ projectIdNotNull: boolean | null + /** Whether any public table carries a `project_id` column. */ + projectColumns: boolean } async function readProjectSchemaState(): Promise { @@ -30,25 +35,29 @@ async function readProjectSchemaState(): Promise { to_regclass('public.project_membership_rollout') IS NOT NULL AS rollout, to_regclass('drizzle.__drizzle_migrations') IS NOT NULL AS journal, (SELECT attnotnull FROM pg_attribute WHERE attrelid = to_regclass('public.workspace') - AND attname = 'project_id' AND NOT attisdropped) AS "projectIdNotNull" + AND attname = 'project_id' AND NOT attisdropped) AS "projectIdNotNull", + EXISTS (SELECT 1 FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid + WHERE c.relnamespace = 'public'::regnamespace AND c.relkind IN ('r', 'p') + AND a.attname = 'project_id' AND NOT a.attisdropped) AS "projectColumns" ` return state } /** * Classifies a development push database that has no connector writer to drain: either it - * predates Projects entirely, or an earlier bootstrap committed the expansion and stopped before - * enforcement. Only the versioned 0406 creates the rollout marker, so a marker that the migration - * journal does not account for, beside a connector nothing has written, is a bootstrap to resume. - * Every other shape returns null and keeps the writer-drain refusal. + * predates Projects entirely (no Project table, connector, marker, or `project_id` column + * anywhere), or an earlier bootstrap committed the expansion and has not finished. Only the + * versioned 0406 creates the rollout marker, so a marker that the migration journal does not + * account for, beside a connector nothing has written, is a bootstrap in progress. Every other + * shape returns null and keeps the writer-drain refusal. */ async function classifyProjectBootstrap( state: ProjectSchemaState ): Promise<'fresh' | 'resume' | null> { - if (!state.workspace || state.projectIdNotNull) return null - const parts = [state.project, state.connector, state.rollout, state.projectIdNotNull !== null] - if (parts.every((present) => !present)) return 'fresh' - if (!parts.every(Boolean)) return null + if (process.env.SIM_DEV_DB_PUSH !== '1' || !state.workspace) return null + if (!state.project && !state.connector && !state.rollout && !state.projectColumns) return 'fresh' + if (!state.project || !state.connector || !state.rollout || state.projectIdNotNull !== false) + return null const [connector] = await sql<{ populated: boolean }[]>` SELECT EXISTS (SELECT 1 FROM public.project_workspace) AS populated` if (connector.populated) return null @@ -70,13 +79,12 @@ async function migrationStatements(tag: string): Promise { } /** - * Applies the versioned Project expansion, then the registered script migration on this - * single-session connection, recording it as the migration runner does. 0406 opens by committing - * the migrator's batch; leaving that out commits the 0394 foundation together with the rollout - * marker, so an interrupted bootstrap leaves either no Project schema or one it can resume. + * Replays the versioned Project expansion. 0406 opens by committing the migrator's batch; leaving + * that out commits the 0394 foundation together with the rollout marker, so an interrupted + * expansion leaves either no Project schema or one the next push resumes. */ -async function bootstrapProjects(mode: 'fresh' | 'resume'): Promise { - logger.info('Bootstrapping Projects for a development push database without them', { mode }) +async function expandProjects(mode: 'fresh' | 'resume'): Promise { + logger.info('Expanding Projects for a development push database without them', { mode }) const foundation = mode === 'fresh' ? await migrationStatements('0394_project_foundation') : [] const [batchCommit, ...expansion] = await migrationStatements('0406_workspace_project_column') if (batchCommit.trim() !== 'COMMIT;') { @@ -89,28 +97,41 @@ async function bootstrapProjects(mode: 'fresh' | 'resume'): Promise { await sql.unsafe('ROLLBACK').catch(() => undefined) throw error } - await runScriptMigrations(sql, [projectMembershipMigration]) } -const prepare = process.argv.includes('--prepare') +function assertProjectMembershipComplete(state: ProjectSchemaState): void { + if (state.workspace && (state.connector || !state.projectIdNotNull || !state.project)) { + throw new Error(REFUSAL) + } +} + +/** + * `--prepare` runs before the first Drizzle push and expands an eligible bootstrap. `push.ts` + * then pushes the schema with `workspace.project_id` still nullable, so the columns the Project + * migration reads exist, and calls `--bootstrap` to run the registered migration on this + * single-session connection, recorded as the migration runner records it. Without a flag, the + * post-push pass installs the native constraints. + */ try { let state = await readProjectSchemaState() - if (prepare && process.env.SIM_DEV_DB_PUSH === '1') { + if (process.argv.includes('--prepare')) { const bootstrap = await classifyProjectBootstrap(state) if (bootstrap) { - await bootstrapProjects(bootstrap) + await expandProjects(bootstrap) state = await readProjectSchemaState() } - } - if (state.workspace && (state.connector || !state.projectIdNotNull || !state.project)) { - throw new Error( - 'Existing Project databases must complete the Project script migration 0032_project_membership before schema push. Use the phased migration path for retained data; schema push cannot bypass the writer-drain requirement' - ) - } - if (!prepare && state.workspace && state.project) { - /** Drizzle cannot express deferred FK timing; fresh push shares the migration finalizer. */ - await enforceProjectMembership(sql) - logger.info('Project membership validation and native constraints completed') + if ((await classifyProjectBootstrap(state)) !== 'resume') assertProjectMembershipComplete(state) + } else if (process.argv.includes('--bootstrap')) { + if ((await classifyProjectBootstrap(state)) !== 'resume') throw new Error(REFUSAL) + await runScriptMigrations(sql, [projectMembershipMigration]) + assertProjectMembershipComplete(await readProjectSchemaState()) + } else { + assertProjectMembershipComplete(state) + if (state.workspace && state.project) { + /** Drizzle cannot express deferred FK timing; fresh push shares the migration finalizer. */ + await enforceProjectMembership(sql) + logger.info('Project membership validation and native constraints completed') + } } } catch (error) { logger.error('Project schema push stopped; resolve the error and retry', { From f51be95cf5a5f1db80329a1d2571d2a5a7313782 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 2 Nov 2026 20:56:18 -0800 Subject: [PATCH 3/3] fix(db): pin the Project bootstrap to the public schema The replayed migration SQL and the script runner name tables unqualified, so a connection search path that lists another schema first created the Project foundation outside public. The refusal snapshot now also covers folders and Projects. --- .../project-push-bootstrap.integration.ts | 16 ++++++++++++++++ .../db/scripts/reconcile-project-membership.ts | 15 +++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/db/scripts/project-push-bootstrap.integration.ts b/packages/db/scripts/project-push-bootstrap.integration.ts index 32a2bffa634..042aebccdd3 100644 --- a/packages/db/scripts/project-push-bootstrap.integration.ts +++ b/packages/db/scripts/project-push-bootstrap.integration.ts @@ -69,6 +69,8 @@ async function retainedState(sql: Sql) { return { schema, workspaces: await sql`SELECT to_jsonb(w) AS row FROM workspace w ORDER BY id`, + folders: await sql`SELECT to_jsonb(f) AS row FROM folder f ORDER BY id`, + projects: schema.project ? await sql`SELECT to_jsonb(p) AS row FROM project p ORDER BY id` : [], connector: schema.connector ? await sql`SELECT * FROM project_workspace ORDER BY workspace_id` : [], @@ -261,6 +263,20 @@ describe('development schema push of a database that predates Projects', () => { }) }, 240_000) + it('bootstraps into public when the connection search path lists another schema first', async () => { + await preProjectDatabase(async (sql, url) => { + await sql`CREATE SCHEMA shadow` + const shadowed = new URL(url) + shadowed.searchParams.set('options', '-c search_path=shadow,public') + const pushed = runPush(shadowed.toString(), true) + expect(pushed.status, pushed.output).toBe(0) + const shadowTables = await sql`SELECT tablename FROM pg_tables WHERE schemaname = 'shadow' + AND tablename IN ('project', 'project_workspace', 'project_membership_rollout')` + await expectBootstrapped(sql) + expect(shadowTables).toEqual([]) + }) + }, 240_000) + it('leaves the database untouched outside development push mode', async () => { await preProjectDatabase(async (sql, url) => { const refused = runPush(url, false) diff --git a/packages/db/scripts/reconcile-project-membership.ts b/packages/db/scripts/reconcile-project-membership.ts index 0495ba4e031..495810d323a 100644 --- a/packages/db/scripts/reconcile-project-membership.ts +++ b/packages/db/scripts/reconcile-project-membership.ts @@ -99,6 +99,17 @@ async function expandProjects(mode: 'fresh' | 'resume'): Promise { } } +/** Migration SQL and the script runner name tables unqualified; pin them to `public`, as the migrator's default session does. */ +async function inPublicSchema(run: () => Promise): Promise { + const [settings] = await sql<{ search_path: string }[]>`SHOW search_path` + await sql`SELECT set_config('search_path', 'public, pg_temp', false)` + try { + await run() + } finally { + await sql`SELECT set_config('search_path', ${settings.search_path}, false)` + } +} + function assertProjectMembershipComplete(state: ProjectSchemaState): void { if (state.workspace && (state.connector || !state.projectIdNotNull || !state.project)) { throw new Error(REFUSAL) @@ -117,13 +128,13 @@ try { if (process.argv.includes('--prepare')) { const bootstrap = await classifyProjectBootstrap(state) if (bootstrap) { - await expandProjects(bootstrap) + await inPublicSchema(() => expandProjects(bootstrap)) state = await readProjectSchemaState() } if ((await classifyProjectBootstrap(state)) !== 'resume') assertProjectMembershipComplete(state) } else if (process.argv.includes('--bootstrap')) { if ((await classifyProjectBootstrap(state)) !== 'resume') throw new Error(REFUSAL) - await runScriptMigrations(sql, [projectMembershipMigration]) + await inPublicSchema(() => runScriptMigrations(sql, [projectMembershipMigration])) assertProjectMembershipComplete(await readProjectSchemaState()) } else { assertProjectMembershipComplete(state)