diff --git a/.changepacks/changepack_log_next_coordinator_client.json b/.changepacks/changepack_log_next_coordinator_client.json new file mode 100644 index 000000000..39afef1c4 --- /dev/null +++ b/.changepacks/changepack_log_next_coordinator_client.json @@ -0,0 +1,7 @@ +{ + "changes": { + "packages/next-plugin/package.json": "Patch" + }, + "note": "Turbopack loaders talk to the coordinator more robustly: the CSS loader keeps a port per port file, so two apps in one process no longer read each other's CSS; a loader whose coordinator restarted reads the new port and sends the request there instead of failing; a port file that is still being written is waited for instead of read as a bad port. New errors: `Coordinator port file names no valid port` and `Coordinator at 127.0.0.1: (port file ) sent no response for 300s` (a coordinator that accepts a request and never answers used to hang the build forever)", + "date": "2026-11-15T00:00:00.000Z" +} diff --git a/packages/next-plugin/src/__tests__/coordinator-client.test.ts b/packages/next-plugin/src/__tests__/coordinator-client.test.ts new file mode 100644 index 000000000..139b26cfa --- /dev/null +++ b/packages/next-plugin/src/__tests__/coordinator-client.test.ts @@ -0,0 +1,200 @@ +import * as fs from 'node:fs' +import { mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs' +import { createServer, type Server } from 'node:http' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it, spyOn } from 'bun:test' + +import { + callCoordinator, + type CoordinatorCall, + type CoordinatorReply, + resetCoordinatorClient, + setCoordinatorResponseTimeoutForTesting, +} from '../coordinator-client' +import { formatPortFile } from '../coordinator-port' + +const dirs: string[] = [] +const servers: Server[] = [] + +function tmp(): string { + const dir = mkdtempSync(join(tmpdir(), 'devup-client-')) + dirs.push(dir) + return dir +} + +/** A coordinator stand-in that answers every request with `body`. */ +async function serve(body: string): Promise { + const server = createServer((req, res) => { + const chunks: Buffer[] = [] + req.on('data', (chunk: Buffer) => chunks.push(chunk)) + req.on('end', () => { + res.writeHead(200, { 'Content-Type': 'text/plain' }) + res.end(`${body}:${req.method}:${req.url}:${Buffer.concat(chunks)}`) + }) + }) + servers.push(server) + await new Promise((done) => server.listen(0, '127.0.0.1', done)) + return (server.address() as { port: number }).port +} + +/** A port that nothing listens on: bound once, then released. */ +async function deadPort(): Promise { + const server = createServer() + await new Promise((done) => server.listen(0, '127.0.0.1', done)) + const port = (server.address() as { port: number }).port + await new Promise((done) => server.close(() => done())) + return port +} + +function call( + portFile: string, + request: CoordinatorCall = { path: '/css', method: 'GET' }, +): Promise { + return new Promise((done, fail) => + callCoordinator(portFile, request, (error, reply) => + reply === undefined ? fail(error) : done(reply), + ), + ) +} + +afterEach(async () => { + resetCoordinatorClient() + for (const server of servers.splice(0)) { + server.closeAllConnections() + await new Promise((done) => server.close(() => done())) + } + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }) +}) + +describe('callCoordinator', () => { + it('keeps a separate port for each port file', async () => { + const dir = tmp() + const first = join(dir, 'first.port') + const second = join(dir, 'second.port') + writeFileSync(first, formatPortFile(await serve('FIRST'))) + writeFileSync(second, formatPortFile(await serve('SECOND'))) + + expect((await call(first)).body).toBe('FIRST:GET:/css:') + expect((await call(second)).body).toBe('SECOND:GET:/css:') + }) + + it('sends a POST body with its status', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + writeFileSync(portFile, formatPortFile(await serve('APP'))) + + expect( + await call(portFile, { path: '/extract', method: 'POST', body: '{}' }), + ).toEqual({ statusCode: 200, body: 'APP:POST:/extract:{}' }) + }) + + it('follows a coordinator that restarted on a new port', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + writeFileSync(portFile, formatPortFile(await serve('OLD'))) + expect((await call(portFile)).body).toBe('OLD:GET:/css:') + + // The old coordinator exits and its replacement publishes a new port. + for (const server of servers.splice(0)) { + server.closeAllConnections() + await new Promise((done) => server.close(() => done())) + } + writeFileSync(portFile, formatPortFile(await serve('NEW'))) + + expect((await call(portFile)).body).toBe('NEW:GET:/css:') + }) + + it('reports a coordinator that still names the port that refused', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + const port = await deadPort() + writeFileSync(portFile, formatPortFile(port)) + + await expect(call(portFile)).rejects.toThrow( + `Coordinator unreachable at 127.0.0.1:${port}`, + ) + }) + + it('reports a restarted coordinator whose new port refuses too', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + writeFileSync(portFile, formatPortFile(await serve('OLD'))) + await call(portFile) + for (const server of servers.splice(0)) { + server.closeAllConnections() + await new Promise((done) => server.close(() => done())) + } + const next = await deadPort() + writeFileSync(portFile, formatPortFile(next)) + + await expect(call(portFile)).rejects.toThrow( + `Coordinator unreachable at 127.0.0.1:${next}`, + ) + }) + + it('reports a coordinator whose port file disappears after it refused', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + writeFileSync(portFile, formatPortFile(await serve('OLD'))) + await call(portFile) + for (const server of servers.splice(0)) { + server.closeAllConnections() + await new Promise((done) => server.close(() => done())) + } + unlinkSync(portFile) + + await expect(call(portFile)).rejects.toThrow('Coordinator unreachable at') + }) + + it('waits for a port file that is not completely written yet', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + writeFileSync(portFile, '') + const port = await serve('LATE') + setTimeout(() => writeFileSync(portFile, formatPortFile(port)), 100) + + expect((await call(portFile)).body).toBe('LATE:GET:/css:') + }) + + it('reports a port file that never names a valid port', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + writeFileSync(portFile, '70000') + + await expect(call(portFile)).rejects.toThrow( + `Coordinator port file ${portFile} names no valid port.`, + ) + }) + + it('reports a port file that cannot be read', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + writeFileSync(portFile, '1234') + const read = spyOn(fs, 'readFileSync').mockImplementation(() => { + throw 'EACCES' + }) + try { + await expect(call(portFile)).rejects.toThrow('EACCES') + } finally { + read.mockRestore() + } + }) + + it('gives up on a coordinator that accepts the request and never answers', async () => { + const dir = tmp() + const portFile = join(dir, 'coordinator.port') + const server = createServer(() => {}) + servers.push(server) + await new Promise((done) => server.listen(0, '127.0.0.1', done)) + const port = (server.address() as { port: number }).port + writeFileSync(portFile, formatPortFile(port)) + setCoordinatorResponseTimeoutForTesting(50) + + await expect(call(portFile)).rejects.toThrow( + `Coordinator at 127.0.0.1:${port} (port file ${portFile}) sent no response for 0.05s.`, + ) + }) +}) diff --git a/packages/next-plugin/src/coordinator-client.ts b/packages/next-plugin/src/coordinator-client.ts new file mode 100644 index 000000000..93a123b6a --- /dev/null +++ b/packages/next-plugin/src/coordinator-client.ts @@ -0,0 +1,207 @@ +import { existsSync, readFileSync } from 'node:fs' +import { Agent, request } from 'node:http' + +import { + isConnectionError, + missingPortFileError, + parsePortFile, + unreachableCoordinatorError, +} from './coordinator-port' + +/** + * How long a request may go without any response from the coordinator. The + * coordinator holds `/css` until extraction settles (at most its own + * `maxWaitMs` per wait, 60s by default), so this is far above any wait it + * means to make: only a coordinator that accepted the connection and then + * stopped answering reaches it. + */ +const DEFAULT_RESPONSE_TIMEOUT_MS = 300_000 +const PORT_FILE_RETRIES = 20 +const PORT_FILE_RETRY_MS = 50 + +let responseTimeoutMs = DEFAULT_RESPONSE_TIMEOUT_MS +const keepAliveAgent = new Agent({ keepAlive: true }) +/** The port each port file named when last read, so each app keeps its own. */ +const cachedPorts = new Map() + +export interface CoordinatorCall { + path: string + method: 'GET' | 'POST' + body?: string +} + +export interface CoordinatorReply { + statusCode: number + body: string +} + +type ReplyCallback = (error: Error | null, reply?: CoordinatorReply) => void + +function isValidPort(port: number): boolean { + return Number.isInteger(port) && port > 0 && port < 65536 +} + +/** + * The port `portFile` names, or `undefined` while it names none: missing, + * or not completely written yet. + */ +function readPort(portFile: string): number | undefined { + const cached = cachedPorts.get(portFile) + if (cached !== undefined) return cached + if (!existsSync(portFile)) return undefined + const { port } = parsePortFile(readFileSync(portFile, 'utf-8')) + if (!isValidPort(port)) return undefined + cachedPorts.set(portFile, port) + return port +} + +function invalidPortFileError(portFile: string): Error { + return new Error( + `Coordinator port file ${portFile} names no valid port. ` + + 'Restart the dev server or build so the coordinator writes it again.', + ) +} + +function noResponseError(portFile: string, port: number): Error { + return new Error( + `Coordinator at 127.0.0.1:${port} (port file ${portFile}) sent no response for ${responseTimeoutMs / 1000}s. ` + + 'It is stuck or another process holds the port: restart the dev server or build.', + ) +} + +/** Wait (about one second) for the port file to name a port. */ +function waitForPort( + portFile: string, + retries: number, + onPort: (port: number) => void, + onError: (error: Error) => void, +): void { + let port: number | undefined + try { + port = readPort(portFile) + } catch (error) { + onError(error instanceof Error ? error : new Error(String(error))) + return + } + if (port !== undefined) { + onPort(port) + return + } + if (retries > 0) { + setTimeout( + () => waitForPort(portFile, retries - 1, onPort, onError), + PORT_FILE_RETRY_MS, + ) + return + } + onError( + existsSync(portFile) + ? invalidPortFileError(portFile) + : missingPortFileError(portFile), + ) +} + +function send( + portFile: string, + port: number, + call: CoordinatorCall, + callback: ReplyCallback, +): void { + let settled = false + const settle: ReplyCallback = (error, reply) => { + if (settled) return + settled = true + callback(error, reply) + } + const req = request( + { + hostname: '127.0.0.1', + port, + path: call.path, + method: call.method, + headers: + call.body === undefined + ? undefined + : { 'Content-Type': 'application/json' }, + agent: keepAliveAgent, + timeout: responseTimeoutMs, + }, + (res) => { + const chunks: Buffer[] = [] + res.on('data', (chunk: Buffer) => chunks.push(chunk)) + res.on('end', () => + settle(null, { + statusCode: res.statusCode ?? 0, + body: Buffer.concat(chunks).toString('utf-8'), + }), + ) + }, + ) + req.on('timeout', () => req.destroy(noResponseError(portFile, port))) + req.on('error', (error) => settle(error)) + if (call.body !== undefined) req.write(call.body) + req.end() +} + +/** + * Call the coordinator that `portFile` names. + * + * A port that refuses the connection is forgotten and the file read again: a + * coordinator that restarted publishes a new port, and the call is sent there + * once. A port that still does not answer, a port file that never names a + * port, and a coordinator that stops answering mid-request are errors that + * say which coordinator and how to recover. + */ +export function callCoordinator( + portFile: string, + call: CoordinatorCall, + callback: ReplyCallback, +): void { + waitForPort( + portFile, + PORT_FILE_RETRIES, + (port) => { + send(portFile, port, call, (error, reply) => { + if (error === null || !isConnectionError(error)) { + callback(error, reply) + return + } + cachedPorts.delete(portFile) + const unreachable = (cause: Error) => { + cachedPorts.delete(portFile) + callback(unreachableCoordinatorError(portFile, cause)) + } + waitForPort( + portFile, + PORT_FILE_RETRIES, + (next) => { + if (next === port) { + unreachable(error) + return + } + send(portFile, next, call, (retryError, retryReply) => { + if (retryError !== null && isConnectionError(retryError)) { + unreachable(retryError) + return + } + callback(retryError, retryReply) + }) + }, + () => unreachable(error), + ) + }) + }, + (error) => callback(error), + ) +} + +/** @internal Forget cached ports and restore the default timeout. */ +export function resetCoordinatorClient(): void { + cachedPorts.clear() + responseTimeoutMs = DEFAULT_RESPONSE_TIMEOUT_MS +} + +/** @internal Shorten the response timeout for tests. */ +export function setCoordinatorResponseTimeoutForTesting(ms: number): void { + responseTimeoutMs = ms +} diff --git a/packages/next-plugin/src/css-loader.ts b/packages/next-plugin/src/css-loader.ts index 5a414c8cd..63cd07f7a 100644 --- a/packages/next-plugin/src/css-loader.ts +++ b/packages/next-plugin/src/css-loader.ts @@ -1,5 +1,4 @@ import { existsSync, readFileSync } from 'node:fs' -import { Agent, request } from 'node:http' import { type DevupCompiler, @@ -8,12 +7,7 @@ import { } from '@devup-ui/plugin-utils' import type { RawLoaderDefinitionFunction } from 'webpack' -import { - isConnectionError, - missingPortFileError, - parsePortFile, - unreachableCoordinatorError, -} from './coordinator-port' +import { callCoordinator, resetCoordinatorClient } from './coordinator-client' import { loadWasm } from './wasm' export interface DevupUICssLoaderOptions { @@ -32,17 +26,8 @@ export interface DevupUICssLoaderOptions { } let initializedCompilers = new WeakSet() -let cachedPort: number | null = null -const keepAliveAgent = new Agent({ keepAlive: true }) - -function readCoordinatorPort(portFile: string): number { - if (cachedPort !== null) return cachedPort - cachedPort = parsePortFile(readFileSync(portFile, 'utf-8')).port - return cachedPort -} - function fetchCssFromCoordinator( - port: number, + portFile: string, fileNum: number | null, importMainCss: boolean, waitForIdle: boolean, @@ -52,28 +37,21 @@ function fetchCssFromCoordinator( if (fileNum != null) params.set('fileNum', String(fileNum)) params.set('importMainCss', String(importMainCss)) if (waitForIdle) params.set('waitForIdle', 'true') - const req = request( - { - hostname: '127.0.0.1', - port, - path: `/css?${params.toString()}`, - method: 'GET', - agent: keepAliveAgent, - }, - (res) => { - const chunks: Buffer[] = [] - res.on('data', (chunk: Buffer) => chunks.push(chunk)) - res.on('end', () => { - if (res.statusCode !== 200) { - callback(new Error(`Coordinator CSS error: ${res.statusCode}`)) - return - } - callback(null, Buffer.concat(chunks).toString('utf-8')) - }) + callCoordinator( + portFile, + { path: `/css?${params.toString()}`, method: 'GET' }, + (error, reply) => { + if (reply === undefined) { + callback(error) + return + } + if (reply.statusCode !== 200) { + callback(new Error(`Coordinator CSS error: ${reply.statusCode}`)) + return + } + callback(null, reply.body) }, ) - req.on('error', (err) => callback(err)) - req.end() } const devupUICssLoader: RawLoaderDefinitionFunction = @@ -102,42 +80,13 @@ const devupUICssLoader: RawLoaderDefinitionFunction = const pathWithQuery = this.resourcePath + resourceQuery const fileNum = getFileNumByFilename(pathWithQuery) const importMainCss = fileNum !== null - const tryFetch = (retries: number) => { - if (!existsSync(coordinatorPortFile)) { - if (retries > 0) { - setTimeout(() => tryFetch(retries - 1), 50) - return - } - callback(missingPortFileError(coordinatorPortFile)) - return - } - try { - const port = readCoordinatorPort(coordinatorPortFile) - fetchCssFromCoordinator( - port, - fileNum, - importMainCss, - !watch, - (err, css) => { - if (err) { - if (isConnectionError(err)) { - // The port may belong to a coordinator that has since been - // replaced; forget it so the next call re-reads the file. - cachedPort = null - return callback( - unreachableCoordinatorError(coordinatorPortFile, err), - ) - } - return callback(err) - } - callback(null, css) - }, - ) - } catch (error) { - callback(error as Error) - } - } - tryFetch(20) + fetchCssFromCoordinator( + coordinatorPortFile, + fileNum, + importMainCss, + !watch, + callback, + ) return } @@ -188,7 +137,7 @@ export default devupUICssLoader /** @internal Reset init state for testing purposes only */ export const resetInit = () => { initializedCompilers = new WeakSet() - cachedPort = null + resetCoordinatorClient() } export { setWasmForTesting } from './wasm' diff --git a/packages/next-plugin/src/loader.ts b/packages/next-plugin/src/loader.ts index 917b43c87..fb9889c43 100644 --- a/packages/next-plugin/src/loader.ts +++ b/packages/next-plugin/src/loader.ts @@ -1,6 +1,5 @@ import { existsSync, readFileSync } from 'node:fs' import { writeFile } from 'node:fs/promises' -import { Agent, request } from 'node:http' import { basename, dirname, join, relative, resolve } from 'node:path' import { @@ -13,12 +12,7 @@ import { import type { RawLoaderDefinitionFunction } from 'webpack' import { takeExtractOutput } from './coordinator' -import { - isConnectionError, - missingPortFileError, - parsePortFile, - unreachableCoordinatorError, -} from './coordinator-port' +import { callCoordinator, resetCoordinatorClient } from './coordinator-client' import { loadWasm } from './wasm' const stateWriter = createStateWriter((path, content, encoding) => @@ -45,9 +39,6 @@ export interface DevupUILoaderOptions { } let initializedCompilers = new WeakSet() -const cachedPorts = new Map() -const keepAliveAgent = new Agent({ keepAlive: true }) - interface CoordinatorResponse { code?: string error?: string @@ -59,15 +50,6 @@ function toLoaderError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)) } -function readCoordinatorPort(portFile: string): number { - const cachedPort = cachedPorts.get(portFile) - if (cachedPort !== undefined) return cachedPort - - const port = parsePortFile(readFileSync(portFile, 'utf-8')).port - cachedPorts.set(portFile, port) - return port -} - function parseCoordinatorResponse(content: string): CoordinatorResponse { const data: unknown = JSON.parse(content) if (typeof data !== 'object' || data === null) { @@ -95,7 +77,7 @@ function parseSourceMap(sourceMap: string | undefined): string | null { } function coordinatorExtract( - port: number, + portFile: string, body: string, callback: ( err: Error | null, @@ -104,42 +86,31 @@ function coordinatorExtract( dependencies?: string[], ) => void, ): void { - const req = request( - { - hostname: '127.0.0.1', - port, - path: '/extract', - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - agent: keepAliveAgent, - }, - (res) => { - const chunks: Buffer[] = [] - res.on('data', (chunk: Buffer) => chunks.push(chunk)) - res.on('end', () => { - try { - const data = parseCoordinatorResponse( - Buffer.concat(chunks).toString('utf-8'), - ) - if (res.statusCode !== 200) { - callback(new Error(data.error ?? 'Coordinator error')) - return - } - if (data.code === undefined) { - callback(new Error('Coordinator response missing code')) - return - } - const sourceMap = parseSourceMap(data.map) - callback(null, data.code, sourceMap, data.dependencies) - } catch (e) { - callback(toLoaderError(e)) + callCoordinator( + portFile, + { path: '/extract', method: 'POST', body }, + (error, reply) => { + if (reply === undefined) { + callback(error) + return + } + try { + const data = parseCoordinatorResponse(reply.body) + if (reply.statusCode !== 200) { + callback(new Error(data.error ?? 'Coordinator error')) + return } - }) + if (data.code === undefined) { + callback(new Error('Coordinator response missing code')) + return + } + const sourceMap = parseSourceMap(data.map) + callback(null, data.code, sourceMap, data.dependencies) + } catch (e) { + callback(toLoaderError(e)) + } }, ) - req.on('error', (err) => callback(err)) - req.write(body) - req.end() } const devupUILoader: RawLoaderDefinitionFunction = @@ -166,60 +137,29 @@ const devupUILoader: RawLoaderDefinitionFunction = if (coordinatorPortFile) { this.addDependency(coordinatorPortFile) const callback = this.async() - const tryCoordinator = (retries: number) => { - if (!existsSync(coordinatorPortFile)) { - if (retries > 0) { - setTimeout(() => tryCoordinator(retries - 1), 50) - return + // POSIX-normalize so the engine's bucket key matches the canonical map + // and FILE_ROUTES keys (both built with forward slashes). Without this, + // canonical collapse and atom hoisting silently no-op on Windows. + const relativePath = relative( + process.cwd(), + this.resourcePath, + ).replaceAll('\\', '/') + const body = JSON.stringify({ + filename: relativePath, + code: source.toString(), + resourcePath: this.resourcePath, + }) + coordinatorExtract( + coordinatorPortFile, + body, + (err, content, sourceMap, dependencies = []) => { + if (err) return callback(err) + for (const dependency of dependencies) { + this.addDependency(resolve(dependency)) } - // Port file never appeared — fall through to error - callback(missingPortFileError(coordinatorPortFile)) - return - } - try { - const port = readCoordinatorPort(coordinatorPortFile) - // POSIX-normalize so the engine's bucket key matches the canonical map - // and FILE_ROUTES keys (both built with forward slashes). Without this, - // canonical collapse and atom hoisting silently no-op on Windows. - const relativePath = relative( - process.cwd(), - this.resourcePath, - ).replaceAll('\\', '/') - const body = JSON.stringify({ - filename: relativePath, - code: source.toString(), - resourcePath: this.resourcePath, - }) - coordinatorExtract( - port, - body, - (err, content, sourceMap, dependencies = []) => { - if (err) { - if (isConnectionError(err)) { - // Forget a port that no longer answers so the next file - // re-reads the port file instead of repeating the timeout. - cachedPorts.delete(coordinatorPortFile) - return callback( - unreachableCoordinatorError(coordinatorPortFile, err), - ) - } - return callback(err) - } - for (const dependency of dependencies) { - this.addDependency(resolve(dependency)) - } - callback( - null, - content, - sourceMap as Parameters[2], - ) - }, - ) - } catch (error) { - callback(toLoaderError(error)) - } - } - tryCoordinator(20) // 20 retries × 50ms = 1s max wait + callback(null, content, sourceMap as Parameters[2]) + }, + ) return } @@ -334,7 +274,7 @@ export default devupUILoader /** @internal Reset init state for testing purposes only */ export const resetInit = () => { initializedCompilers = new WeakSet() - cachedPorts.clear() + resetCoordinatorClient() } export { setWasmForTesting } from './wasm'