diff --git a/.changepacks/changepack_log_next_transform_cache.json b/.changepacks/changepack_log_next_transform_cache.json new file mode 100644 index 000000000..992878f9d --- /dev/null +++ b/.changepacks/changepack_log_next_transform_cache.json @@ -0,0 +1 @@ +{"changes":{"packages/next-plugin/package.json":"Patch"},"note":"Answer a repeated Turbopack extraction of an unchanged module (the server and client compiles of one file) from the coordinator's per-file transform cache instead of running the engine again","date":"2026-11-15T00:00:00.000Z"} diff --git a/packages/next-plugin/src/__tests__/coordinator.test.ts b/packages/next-plugin/src/__tests__/coordinator.test.ts index 4360975fc..487aab477 100644 --- a/packages/next-plugin/src/__tests__/coordinator.test.ts +++ b/packages/next-plugin/src/__tests__/coordinator.test.ts @@ -1735,6 +1735,119 @@ describe('coordinator per-bucket completion', () => { }) }) +describe('coordinator transform cache', () => { + function extractResult(code: string, dependencies: string[] = []) { + return { + code, + map: undefined, + cssFile: 'devup-ui-1.css', + updatedBaseStyle: false, + dependencies, + free: mock(), + [Symbol.dispose]: mock(), + } + } + + async function start() { + const coordinator = startCoordinator(makeOptions()) + await new Promise((r) => setTimeout(r, 100)) + const port = parseInt( + (writeFileSyncSpy.mock.calls[0] as [string, string])[1], + ) + return { coordinator, port } + } + + async function extract( + port: number, + code: string, + resourcePath = join(process.cwd(), 'src', 'A.tsx'), + ) { + const res = await httpRequest( + port, + 'POST', + '/extract', + JSON.stringify({ filename: 'src/A.tsx', code, resourcePath }), + ) + return { status: res.status, ...JSON.parse(res.body) } + } + + it('answers the second compile of an unchanged module without WASM', async () => { + getCssSpy.mockReturnValue('css') + exportSheetSpy.mockReturnValue('{}') + exportClassMapSpy.mockReturnValue('{}') + exportFileMapSpy.mockReturnValue('{}') + codeExtractSpy.mockImplementation((_filename: string, code: string) => + extractResult(`out:${code}`), + ) + const { coordinator, port } = await start() + + // Server and client graphs compile the same source. + expect((await extract(port, 'v1')).code).toBe('out:v1') + const writes = writeFileSpy.mock.calls.length + expect(await extract(port, 'v1')).toMatchObject({ + status: 200, + code: 'out:v1', + cssFile: 'devup-ui-1.css', + }) + expect(codeExtractSpy).toHaveBeenCalledTimes(1) + // Nothing changed, so nothing is written again. + expect(writeFileSpy.mock.calls.length).toBe(writes) + + // An edit extracts again, and so does going back: the sheet then holds + // the edit's styles, not the first source's. + await extract(port, 'v2') + await extract(port, 'v1') + expect(codeExtractSpy).toHaveBeenCalledTimes(3) + + // The CSS directory is part of the output. + await extract(port, 'v1', join(process.cwd(), 'src', 'nested', 'A.tsx')) + expect(codeExtractSpy).toHaveBeenCalledTimes(4) + + coordinator.close() + }) + + it('extracts again once a file the extraction read changes', async () => { + const tokens = join(tmpDir, 'cache-tokens.ts') + const later = join(tmpDir, 'cache-later.ts') + await Bun.write(tokens, 'export const a = 1') + codeExtractSpy.mockImplementation(() => + extractResult('out', [tokens, later]), + ) + const { coordinator, port } = await start() + + await extract(port, 'src') + await extract(port, 'src') + expect(codeExtractSpy).toHaveBeenCalledTimes(1) + + await Bun.write(tokens, 'export const a = 22') + await extract(port, 'src') + expect(codeExtractSpy).toHaveBeenCalledTimes(2) + + // A dependency that did not exist and now does counts as a change too. + await Bun.write(later, 'export const b = 1') + await extract(port, 'src') + expect(codeExtractSpy).toHaveBeenCalledTimes(3) + + coordinator.close() + }) + + it('never answers from a cache entry an extraction error replaced', async () => { + codeExtractSpy.mockImplementation(() => extractResult('out')) + const { coordinator, port } = await start() + + await extract(port, 'src') + codeExtractSpy.mockImplementationOnce(() => { + throw new Error('parse error') + }) + // Same filename, new source that fails. + expect((await extract(port, 'broken')).status).toBe(500) + await extract(port, 'src') + expect(codeExtractSpy).toHaveBeenCalledTimes(3) + + coordinator.close() + }) +}) + describe('coordinator theme reload', () => { async function start(theme: CoordinatorOptions['theme']) { const coordinator = startCoordinator(makeOptions({ theme })) @@ -1795,6 +1908,50 @@ describe('coordinator theme reload', () => { coordinator.close() }) + it('extracts an unchanged module again once the theme reloads', async () => { + codeExtractSpy.mockImplementation(() => ({ + code: 'code', + map: undefined, + cssFile: undefined, + updatedBaseStyle: false, + dependencies: [], + free: mock(), + [Symbol.dispose]: mock(), + })) + getCssSpy.mockReturnValue('css') + const devupFile = join(tmpDir, 'theme-cached.json') + await Bun.write(devupFile, '{}') + // A source that exists, so a repeated extraction keeps its cache entry + const resourcePath = join(tmpDir, 'theme-cached.tsx') + await Bun.write(resourcePath, 'c') + const reload = mock(() => [devupFile]) + const { coordinator, port } = await start({ files: [devupFile], reload }) + const extractSource = () => + httpRequest( + port, + 'POST', + '/extract', + JSON.stringify({ filename: 'src/A.tsx', code: 'c', resourcePath }), + ) + + await extractSource() + await extractSource() + expect(codeExtractSpy).toHaveBeenCalledTimes(1) + + // The cached output was made under the old theme, so it goes + await Bun.write(devupFile, '{"theme":{"colors":{"primary":"red"}}}') + expect((await httpRequest(port, 'GET', '/css')).status).toBe(200) + expect(reload).toHaveBeenCalledTimes(1) + expect((await extractSource()).status).toBe(200) + expect(codeExtractSpy).toHaveBeenCalledTimes(2) + + // and the new theme's output is cached again + await extractSource() + expect(codeExtractSpy).toHaveBeenCalledTimes(2) + + coordinator.close() + }) + it('reports a theme that fails to load and retries on the next request', async () => { const devupFile = join(tmpDir, 'theme-broken.json') await Bun.write(devupFile, '{}') @@ -1896,4 +2053,55 @@ describe('coordinator deleted sources', () => { expect(writeFileSpy).not.toHaveBeenCalled() coordinator.close() }) + + it('extracts a deleted file again when it comes back unchanged', async () => { + // Given: two cached extractions, one of whose files is then deleted + using removeFile = spyOn(wasm, 'removeFile').mockReturnValue(true) + const deleted = new Set() + using _existsSync = spyOn(fs, 'existsSync').mockImplementation( + (path) => !deleted.has(String(path)), + ) + codeExtractSpy.mockImplementation(() => ({ + code: 'code', + map: undefined, + cssFile: undefined, + updatedBaseStyle: false, + free: mock(), + [Symbol.dispose]: mock(), + })) + getCssSpy.mockReturnValue('base') + exportSheetSpy.mockReturnValue('sheet') + exportClassMapSpy.mockReturnValue('classes') + exportFileMapSpy.mockReturnValue('files') + const coordinator = startCoordinator(makeOptions()) + await new Promise((r) => setTimeout(r, 100)) + const port = parseInt( + (writeFileSyncSpy.mock.calls[0] as [string, string])[1], + ) + const extract = (filename: string) => + httpRequest( + port, + 'POST', + '/extract', + JSON.stringify({ + filename, + code: 'c', + resourcePath: join(tmpDir, filename), + }), + ) + await extract('src/Gone.tsx') + await extract('src/App.tsx') + deleted.add(join(tmpDir, 'src/Gone.tsx')) + await extract('src/App.tsx') + expect(removeFile.mock.calls).toEqual([['src/Gone.tsx']]) + expect(codeExtractSpy).toHaveBeenCalledTimes(2) + + // When: the file is restored with the same source + deleted.clear() + await extract('src/Gone.tsx') + + // Then: its styles left the sheet, so it is extracted again + expect(codeExtractSpy).toHaveBeenCalledTimes(3) + coordinator.close() + }) }) diff --git a/packages/next-plugin/src/coordinator.ts b/packages/next-plugin/src/coordinator.ts index 3f6670ed7..e5b928bab 100644 --- a/packages/next-plugin/src/coordinator.ts +++ b/packages/next-plugin/src/coordinator.ts @@ -1,6 +1,6 @@ import { existsSync, statSync, writeFile, writeFileSync } from 'node:fs' import { createServer, type IncomingMessage, type Server } from 'node:http' -import { basename, dirname, join, relative } from 'node:path' +import { basename, dirname, join, relative, resolve } from 'node:path' import { getFileNumByFilename } from '@devup-ui/plugin-utils' @@ -197,6 +197,22 @@ function readBody(req: IncomingMessage): Promise { let server: Server | null = null +// The latest extraction of each file. Next compiles a module that both the +// server and the client graph import once per graph, with the same source, +// and in no guaranteed order. Re-running WASM for the second copy only +// rebuilds the same styles, so it answers from here instead. Keeping only the +// latest source per file bounds the cache by the project's file count and +// guarantees the sheet still holds exactly that source's styles. +interface CachedExtract { + code: string + relCssDir: string + /** mtime and size of the files the extraction read through the resolver */ + dependencies: string + output: ExtractOutputSnapshot +} + +const transformCache = new Map() + interface ThemeWatch { files: string[] stamp: string @@ -205,12 +221,15 @@ interface ThemeWatch { let themeWatch: ThemeWatch | undefined -/** What changes when a theme input is edited, added or deleted. */ -function themeStamp(files: string[]): string { +/** + * What changes when one of the files is edited, added or deleted: a theme + * input, or a file an extraction read through the resolver. + */ +function fileStamp(files: string[] = []): string { return files .map((file) => { try { - const stats = statSync(file) + const stats = statSync(resolve(file)) return `${stats.mtimeMs}:${stats.size}` } catch { return 'missing' @@ -219,12 +238,28 @@ function themeStamp(files: string[]): string { .join('|') } +function cachedExtract( + filename: string, + code: string, + relCssDir: string, +): ExtractOutputSnapshot | undefined { + const cached = transformCache.get(filename) + return cached?.code === code && + cached.relCssDir === relCssDir && + cached.dependencies === fileStamp(cached.output.dependencies) + ? cached.output + : undefined +} + /** Register the theme again when one of its input files has changed. */ function refreshTheme(): void { - if (!themeWatch || themeStamp(themeWatch.files) === themeWatch.stamp) return + if (!themeWatch || fileStamp(themeWatch.files) === themeWatch.stamp) return + // Every cached extraction was made under the old theme. Even a reload that + // throws may have changed the registered theme, so drop them first. + transformCache.clear() // A throwing reload keeps the old stamp, so the next request retries. const files = themeWatch.reload() - themeWatch = { ...themeWatch, files, stamp: themeStamp(files) } + themeWatch = { ...themeWatch, files, stamp: fileStamp(files) } } // Extraction tracking for waitForIdle. @@ -449,9 +484,10 @@ export function startCoordinator(options: CoordinatorOptions): { for (const file of options.prewarmedFiles ?? []) extractedFiles.add(file) extractedSources.clear() fileNumToBucket.clear() + transformCache.clear() themeWatch = options.theme && { ...options.theme, - stamp: themeStamp(options.theme.files), + stamp: fileStamp(options.theme.files), } // A deleted file is never extracted again, so its styles would stay in the @@ -462,6 +498,8 @@ export function startCoordinator(options: CoordinatorOptions): { for (const [filename, resourcePath] of extractedSources) { if (existsSync(resourcePath)) continue extractedSources.delete(filename) + // Its styles leave the sheet, so the same source must extract again. + transformCache.delete(filename) if (removeFile(filename)) removed = true } if (!removed) return @@ -562,22 +600,37 @@ export function startCoordinator(options: CoordinatorOptions): { // Require exact source equality because Turbopack may hand a loader // code modified by an earlier transform. const prewarmed = prewarmedOutputs.get(filename) - const cacheHit = prewarmed?.source === code const extractStartedAt = profileStart() - const result = cacheHit - ? prewarmed - : takeExtractOutput( - extract( - filename, - code, - libPackage, - relCssDir, - singleCss, - false, - true, - importAliases, - ), - ) + const cached = + prewarmed?.source === code + ? prewarmed + : cachedExtract(filename, code, relCssDir) + const cacheHit = cached !== undefined + let result: ExtractOutputSnapshot + if (cached) { + result = cached + } else { + // Whatever the engine kept of an earlier source is gone if this throws. + transformCache.delete(filename) + result = takeExtractOutput( + extract( + filename, + code, + libPackage, + relCssDir, + singleCss, + false, + true, + importAliases, + ), + ) + transformCache.set(filename, { + code, + relCssDir, + dependencies: fileStamp(result.dependencies), + output: result, + }) + } const extractDurationMs = elapsedMs(extractStartedAt) // When singleCss=false, rewrite per-file CSS imports so Turbopack can resolve them. @@ -804,6 +857,7 @@ export const resetCoordinator = () => { bucketToMembers = new Map() canonicalMapRef = {} expectedBaseFiles = new Set() + transformCache.clear() themeWatch = undefined writeChain.clear() latestContent.clear()