From c116dae48db9af60873f9709ebfb35c966890cc8 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 21 Oct 2026 14:15:52 -0700 Subject: [PATCH 1/9] fix(targets): report invalid builds and reject output collisions Keep hook pointers and custom marketplace validation consistent with emitted files. Refuse same-target overwrites before writing output, and report package skill, Codex entry, and omitted hooks problems during build. --- src/agent-plugins.ts | 87 ++++++++---- src/package-clients.ts | 8 +- src/targets/agent-plugins.ts | 6 + src/targets/claude.ts | 11 +- src/targets/codex.ts | 70 ++++++---- src/targets/copilot.ts | 3 +- src/targets/cursor.ts | 11 +- src/targets/engine.ts | 104 +++++++++++++-- src/targets/shared.ts | 25 ++++ src/targets/types.ts | 5 + tests/target-preflight.test.ts | 234 +++++++++++++++++++++++++++++++++ 11 files changed, 484 insertions(+), 80 deletions(-) create mode 100644 tests/target-preflight.test.ts diff --git a/src/agent-plugins.ts b/src/agent-plugins.ts index e8be67c..3d5ae85 100644 --- a/src/agent-plugins.ts +++ b/src/agent-plugins.ts @@ -70,6 +70,7 @@ export function buildPackageManifest({ version, pluginName, pluginConfig, + pluginFiles, }: ManifestBuildContext): Record { const problem = packageNameProblem(pluginName); if (problem) { @@ -77,6 +78,19 @@ export function buildPackageManifest({ `Plugin "${pluginName}" can't be an Agent Plugins package: ${problem}`, ); } + for (const [file, contents] of pluginFiles) { + const skill = /^skills\/([^/]+)\/SKILL\.md$/.exec(file); + if (!skill) { + continue; + } + const issues: ValidationIssue[] = []; + validateSkillContents(contents.toString(), skill[1], pluginName, issues); + for (const issue of issues) { + console.warn( + `Warning: Plugin "${pluginName}" ${file}: ${issue.message} Fix the YAML "name" and "description" fields and make the name match its directory before publishing.`, + ); + } + } return stripUndefined({ $schema: AGENT_PLUGINS_SCHEMA, name: pluginName, @@ -359,40 +373,55 @@ async function validateSkills( if (!(await exists(skillFile))) { continue; } - const where = `${pluginName}: skill "skills/${entry.name}"`; - let frontmatter: Record; - try { - frontmatter = parseMatter(await fs.readFile(skillFile, "utf8")).data; - } catch (err) { + validateSkillContents( + await fs.readFile(skillFile, "utf8"), + entry.name, + pluginName, + issues, + ); + } +} + +/** Shared by build-time diagnostics and on-disk validation. */ +function validateSkillContents( + contents: string, + directoryName: string, + pluginName: string, + issues: ValidationIssue[], +): void { + const where = `${pluginName}: skill "skills/${directoryName}"`; + let frontmatter: Record; + try { + frontmatter = parseMatter(contents).data; + } catch (err) { + error( + issues, + `${where} has unparseable frontmatter: ${(err as Error).message}`, + ); + return; + } + const { name, description } = frontmatter; + if (typeof name !== "string" || !name) { + error(issues, `${where} is missing the required "name" field.`); + } else { + const problem = skillNameProblem(name); + if (problem) { error( issues, - `${where} has unparseable frontmatter: ${(err as Error).message}`, + `${where} has name "${name}", which ${problem} (Agent Skills specification) — conforming clients skip it.`, + ); + } else if (name !== directoryName) { + error( + issues, + `${where} has name "${name}", which must match its directory name "${directoryName}" — conforming clients skip it.`, ); - continue; - } - const { name, description } = frontmatter; - if (typeof name !== "string" || !name) { - error(issues, `${where} is missing the required "name" field.`); - } else { - const problem = skillNameProblem(name); - if (problem) { - error( - issues, - `${where} has name "${name}", which ${problem} (Agent Skills specification) — conforming clients skip it.`, - ); - } else if (name !== entry.name) { - error( - issues, - `${where} has name "${name}", which must match its directory name "${entry.name}" — conforming clients skip it.`, - ); - } - } - if (typeof description !== "string" || !description) { - error(issues, `${where} is missing the required "description" field.`); - } else if (description.length > 1024) { - error(issues, `${where} "description" exceeds 1024 characters.`); } } + if (typeof description !== "string" || !description) { + error(issues, `${where} is missing the required "description" field.`); + } else if (description.length > 1024) { + error(issues, `${where} "description" exceeds 1024 characters.`); + } } /** Spec §5.5. Returns a description of the violation, or `undefined`. */ diff --git a/src/package-clients.ts b/src/package-clients.ts index 6cee1d7..61569b9 100644 --- a/src/package-clients.ts +++ b/src/package-clients.ts @@ -5,7 +5,11 @@ import { PACKAGE_CLIENT_NAMES, type PackageClientName, } from "./schema.js"; -import { codexMarketplace, validateCodexEntry } from "./targets/codex.js"; +import { + buildCodexMarketplaceEntry, + codexMarketplace, + validateCodexEntry, +} from "./targets/codex.js"; import { stripUndefined } from "./targets/shared.js"; import { error, @@ -71,7 +75,7 @@ const codexClient: PackageClient = { placeContent: () => undefined, marketplacePath: path.join(".agents", "plugins", "marketplace.json"), buildMarketplaceEntry: (ctx) => - codexMarketplace.buildMarketplaceEntry(ctx) as Record, + buildCodexMarketplaceEntry(ctx, "agent-plugins"), buildMarketplaceManifest: (ctx) => codexMarketplace.buildMarketplaceManifest(ctx) as Record, validate: async (root, packageDirs, issues) => { diff --git a/src/targets/agent-plugins.ts b/src/targets/agent-plugins.ts index d4b00ab..1f50759 100644 --- a/src/targets/agent-plugins.ts +++ b/src/targets/agent-plugins.ts @@ -134,6 +134,12 @@ function withClients( ); return { ...base, + omittedContentWarnings: clients.some((client) => client.name === "codex") + ? { + hooks: + 'The agent-plugins target does not ship Codex hooks. Keep the codex target if hooks are required, or exclude "hooks" explicitly in pluginpack.config.ts.', + } + : undefined, defaultComponents: [ ...new Set([...base.defaultComponents, ...clientKinds]), ], diff --git a/src/targets/claude.ts b/src/targets/claude.ts index 31c2cc1..ad468fc 100644 --- a/src/targets/claude.ts +++ b/src/targets/claude.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { stripUndefined } from "./shared.js"; +import { marketplaceDirectory, stripUndefined } from "./shared.js"; import { error, readJson, @@ -99,12 +99,13 @@ export const claude: PluginTargetDefinition = { pluginNamePattern, ), - validateOutput: async (root, issues) => { - const marketplacePath = path.join( + validateOutput: async (root, issues, managedPaths = []) => { + const marketplaceDir = marketplaceDirectory( root, ".claude-plugin", - "marketplace.json", + managedPaths, ); + const marketplacePath = path.join(root, marketplaceDir, "marketplace.json"); const ownedPaths = [marketplacePath]; const marketplace = await readJson( marketplacePath, @@ -135,7 +136,7 @@ export const claude: PluginTargetDefinition = { const pluginDir = path.join(root, entry.source); ownedPaths.push(pluginDir); const manifest = await readJson( - path.join(pluginDir, ".claude-plugin", "plugin.json"), + path.join(pluginDir, marketplaceDir, "plugin.json"), `${pluginName} plugin manifest`, issues, ); diff --git a/src/targets/codex.ts b/src/targets/codex.ts index ddba33a..9573e27 100644 --- a/src/targets/codex.ts +++ b/src/targets/codex.ts @@ -18,7 +18,10 @@ import { validateReferencedManifestPaths, } from "./validation-shared.js"; import type { ValidationIssue } from "../types.js"; -import type { PluginTargetDefinition } from "./types.js"; +import type { + MarketplaceEntryContext, + PluginTargetDefinition, +} from "./types.js"; const pluginNamePattern = /^[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?$/; const sourceKinds = new Set(["local", "url", "git-subdir", "npm"]); @@ -102,23 +105,24 @@ export function validateCodexEntry( } else { error(issues, `${name}: source must be a string or a structured object.`); } - const policy = entry.policy as Record | undefined; - if (!policy || typeof policy.installation !== "string") { - error( - issues, - `${name}: entry is missing required field "policy.installation".`, - ); + for (const problem of codexEntryFieldProblems(entry)) { + error(issues, `${name}: ${problem}`); } - if (!policy || typeof policy.authentication !== "string") { - error( - issues, - `${name}: entry is missing required field "policy.authentication".`, - ); + return name; +} + +function codexEntryFieldProblems(entry: Record): string[] { + const problems: string[] = []; + const policy = entry.policy as Record | undefined; + for (const field of ["installation", "authentication"]) { + if (!policy || typeof policy[field] !== "string") { + problems.push(`entry is missing required field "policy.${field}".`); + } } if (typeof entry.category !== "string" || !entry.category) { - error(issues, `${name}: entry is missing required field "category".`); + problems.push('entry is missing required field "category".'); } - return name; + return problems; } /** @@ -174,6 +178,31 @@ function requireName( } } +/** Builds a Codex entry and reports required fields before output is written. */ +export function buildCodexMarketplaceEntry( + { pluginName, pluginPath, pluginConfig, manifest }: MarketplaceEntryContext, + target: "codex" | "agent-plugins" = "codex", +): Record { + const packageClient = target === "agent-plugins"; + const field = `targets.${target}.plugins.${pluginName}.${packageClient ? "clients.codex.entry" : "entry"}`; + const entry = packageClient + ? pluginConfig.clients?.codex?.entry + : pluginConfig.entry; + const problems = codexEntryFieldProblems(entry ?? {}); + if (problems.length > 0) { + console.warn( + `Warning: pluginpack.config.ts ${field}: ${problems.join(" ")} Set the required fields in ${field} before publishing.`, + ); + } + return stripUndefined({ + name: pluginName, + source: `./${pluginPath}`, + description: + pluginConfig.description ?? (manifest?.description as string | undefined), + version: pluginConfig.version ?? (manifest?.version as string | undefined), + }); +} + /** * Shared by both Codex layouts, and by the `agent-plugins` target's Codex * client: everything outside the plugin directory. @@ -190,17 +219,8 @@ export const codexMarketplace = { // Author-supplied `policy`/`category` land here via the per-plugin `entry` // passthrough (see engine.ts's deepMerge) — pluginpack has no way to infer // installation/authentication policy on its own, so the base entry stays - // guess-free and validateOutput errors clearly if they're never supplied. - buildMarketplaceEntry: ({ pluginName, pluginPath, pluginConfig, manifest }) => - stripUndefined({ - name: pluginName, - source: `./${pluginPath}`, - description: - pluginConfig.description ?? - (manifest?.description as string | undefined), - version: - pluginConfig.version ?? (manifest?.version as string | undefined), - }), + // guess-free. Build warns and validateOutput errors if they are absent. + buildMarketplaceEntry: buildCodexMarketplaceEntry, buildMarketplaceManifest: ({ project, plugins }) => stripUndefined({ diff --git a/src/targets/copilot.ts b/src/targets/copilot.ts index cb53387..40da650 100644 --- a/src/targets/copilot.ts +++ b/src/targets/copilot.ts @@ -33,6 +33,7 @@ export const copilot: PluginTargetDefinition = { pluginName, pluginConfig, componentDirs, + pluginFiles, mcpServers, }) => { const manifest: Record = { @@ -53,7 +54,7 @@ export const copilot: PluginTargetDefinition = { if (componentDirs.has("skills")) { manifest.skills = "skills/"; } - if (componentDirs.has("hooks")) { + if (pluginFiles.has("hooks/hooks.json")) { manifest.hooks = "hooks/hooks.json"; } if (mcpServers) { diff --git a/src/targets/cursor.ts b/src/targets/cursor.ts index 4038c46..f064051 100644 --- a/src/targets/cursor.ts +++ b/src/targets/cursor.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import { stripUndefined, titleCase } from "./shared.js"; +import { marketplaceDirectory, stripUndefined, titleCase } from "./shared.js"; import { error, readJson, @@ -165,12 +165,13 @@ export const cursor: PluginTargetDefinition = { pluginNamePattern, ), - validateOutput: async (root, issues) => { - const marketplacePath = path.join( + validateOutput: async (root, issues, managedPaths = []) => { + const marketplaceDir = marketplaceDirectory( root, ".cursor-plugin", - "marketplace.json", + managedPaths, ); + const marketplacePath = path.join(root, marketplaceDir, "marketplace.json"); const ownedPaths = [marketplacePath]; const marketplace = await readJson( marketplacePath, @@ -201,7 +202,7 @@ export const cursor: PluginTargetDefinition = { const pluginDir = path.join(root, entry.source); ownedPaths.push(pluginDir); const manifest = await readJson( - path.join(pluginDir, ".cursor-plugin", "plugin.json"), + path.join(pluginDir, marketplaceDir, "plugin.json"), `${pluginName} plugin manifest`, issues, ); diff --git a/src/targets/engine.ts b/src/targets/engine.ts index 93c2d5c..2db40e9 100644 --- a/src/targets/engine.ts +++ b/src/targets/engine.ts @@ -2,7 +2,7 @@ import { promises as fs } from "node:fs"; import path from "node:path"; import { collectPluginFiles, resolveMcpServers } from "../render.js"; import { readAuthoredPlugin } from "../source.js"; -import { isSafeRelativePath, json, toPosix } from "../fs.js"; +import { isSafeRelativePath, json, toPosix, walkFiles } from "../fs.js"; import { renderMcpConfig } from "../mcp.js"; import { resolvePartials } from "../partials.js"; import { validateNoSurvivingPartialTags } from "./validation-shared.js"; @@ -102,6 +102,19 @@ export async function emitFromDefinition( targetDefinition.forConfig?.(targetConfig) ?? targetDefinition; const version = targetConfig.version ?? project.config.version; const files = new Map(); + const writtenBy = new Map(); + const putFile = (file: string, value: FileValue, owner: string): void => { + const normalized = toPosix(path.normalize(file)); + const key = normalized.normalize("NFC").toLowerCase(); + const earlier = writtenBy.get(key); + if (earlier !== undefined) { + throw new Error( + `Target "${target}": ${earlier} and ${owner} collide at "${normalized}". Change the plugin path or remove the conflicting file in pluginpack.config.ts.`, + ); + } + writtenBy.set(key, owner); + files.set(normalized, value); + }; const entries: Record[] = []; const clientMarketplaces = definition.clientMarketplaces?.(targetConfig) ?? []; @@ -123,6 +136,13 @@ export async function emitFromDefinition( pluginConfig, targetConfig, ); + await warnOmittedContent( + project, + target, + definition, + pluginName, + pluginConfig, + ); let pluginFiles: Map; let mcpServers: Record | undefined; let sourceMetadata: Record | undefined; @@ -179,13 +199,16 @@ export async function emitFromDefinition( const sourceHooksFile = definition.placeFile ? undefined : pluginFiles.get("hooks/hooks.json"); - if (sourceHooksFile !== undefined) { - pluginFiles.delete("hooks/hooks.json"); - } // Placement can map two source files to one path (e.g. agents/x.md and // agents/x.agent.md for Copilot); fail instead of keeping either. const placedFrom = new Map(); for (const [relativePath, value] of pluginFiles) { + if ( + sourceHooksFile !== undefined && + relativePath === "hooks/hooks.json" + ) { + continue; + } for (const placed of definition.placeFile?.(relativePath, pluginName) ?? [ relativePath, ]) { @@ -198,17 +221,25 @@ export async function emitFromDefinition( ); } placedFrom.set(outputPath, relativePath); - files.set(outputPath, value); + putFile( + outputPath, + value, + `plugin "${pluginName}" file "${relativePath}"`, + ); } } if (sourceHooksFile !== undefined) { - files.set(toPosix(definition.hooksPath(pluginPath)), sourceHooksFile); + putFile( + definition.hooksPath(pluginPath), + sourceHooksFile, + `plugin "${pluginName}" hooks`, + ); } const mcpConfigPath = definition.mcpConfigPath(pluginPath); if (mcpServers && mcpConfigPath) { - files.set( - toPosix(mcpConfigPath), + putFile( + mcpConfigPath, json( renderMcpConfig( mcpServers, @@ -216,6 +247,7 @@ export async function emitFromDefinition( `Target "${target}" plugin "${pluginName}"`, ), ), + `plugin "${pluginName}" MCP config`, ); } @@ -230,6 +262,7 @@ export async function emitFromDefinition( pluginName, pluginConfig, componentDirs, + pluginFiles, mcpServers, }); const merged = stripUndefined( @@ -242,7 +275,7 @@ export async function emitFromDefinition( pluginPath, targetConfig, )) { - files.set(toPosix(manifestPath), manifestContent); + putFile(manifestPath, manifestContent, `plugin "${pluginName}" manifest`); } const entryContext = { @@ -281,8 +314,8 @@ export async function emitFromDefinition( targetConfig.clients as Record }> | undefined )?.[marketplace.client]?.manifest; - files.set( - toPosix(marketplace.path), + putFile( + marketplace.path, json( stripUndefined( deepMerge( @@ -296,6 +329,7 @@ export async function emitFromDefinition( ), ), ), + `client "${marketplace.client}" marketplace`, ); } @@ -312,7 +346,7 @@ export async function emitFromDefinition( ); const marketplaceContent = json(marketplace); for (const marketplacePath of definition.marketplacePaths(targetConfig)) { - files.set(toPosix(marketplacePath), marketplaceContent); + putFile(marketplacePath, marketplaceContent, "generated marketplace"); } return artifact(target, outDir, files); @@ -328,7 +362,11 @@ export async function validateFromDefinition( definition: PluginTargetDefinition, managedPaths: string[] = [], ): Promise { - const ownedPaths = await definition.validateOutput(root, issues); + const ownedPaths = await definition.validateOutput( + root, + issues, + managedPaths, + ); await validateNoSurvivingPartialTags( root, [...ownedPaths, ...managedPaths], @@ -336,6 +374,46 @@ export async function validateFromDefinition( ); } +async function warnOmittedContent( + project: ResolvedProject, + target: TargetName, + definition: PluginTargetDefinition, + pluginName: string, + pluginConfig: EmittedPluginConfig, +): Promise { + for (const [kind, message] of Object.entries( + definition.omittedContentWarnings ?? {}, + )) { + // An explicit exclusion is intentional, not a silent migration loss. + if ( + pluginConfig.exclude?.includes(kind) || + (pluginConfig.include && !pluginConfig.include.includes(kind)) || + (pluginConfig.components && !pluginConfig.components.includes(kind)) + ) { + continue; + } + const roots = pluginConfig.source + ? [ + pluginConfig.source, + ...(pluginConfig.overrides ? [pluginConfig.overrides] : []), + ] + : (pluginConfig.from ?? []).flatMap((id) => { + const plugin = project.plugins.get(id); + return plugin + ? [plugin.componentRoots?.[kind] ?? path.join(plugin.dir, kind)] + : []; + }); + for (const source of roots) { + const dir = pluginConfig.source ? path.join(source, kind) : source; + if ((await walkFiles(path.resolve(project.rootDir, dir))).length > 0) { + console.warn( + `Warning: Target "${target}" plugin "${pluginName}": "${toPosix(path.relative(project.rootDir, path.resolve(project.rootDir, dir)))}" is omitted. ${message}`, + ); + } + } + } +} + function emittedPluginMetadata( project: ResolvedProject, pluginConfig: EmittedPluginConfig, diff --git a/src/targets/shared.ts b/src/targets/shared.ts index b0eca5f..128df38 100644 --- a/src/targets/shared.ts +++ b/src/targets/shared.ts @@ -1,3 +1,28 @@ +import path from "node:path"; + +/** + * Recovers a configured marketplaceDir from this target's managed files. + * A custom directory holds both the marketplace and each plugin manifest. + * With no managed manifest, validate the host's standard layout as before. + */ +export function marketplaceDirectory( + root: string, + defaultDir: string, + managedPaths: string[], +): string { + const candidates = managedPaths + .filter((file) => path.basename(file) === "marketplace.json") + .map((file) => path.relative(root, path.dirname(file))) + .filter((dir) => + managedPaths.some((file) => file.endsWith(path.join(dir, "plugin.json"))), + ); + return candidates.includes(defaultDir) + ? defaultDir + : candidates.length === 1 + ? candidates[0] + : defaultDir; +} + /** Deletes keys whose value is `undefined`, mutating and returning `value`. */ export function stripUndefined>(value: T): T { for (const key of Object.keys(value)) { diff --git a/src/targets/types.ts b/src/targets/types.ts index 943c222..8cff75c 100644 --- a/src/targets/types.ts +++ b/src/targets/types.ts @@ -16,6 +16,7 @@ export type ManifestBuildContext = { pluginName: string; pluginConfig: EmittedPluginConfig; componentDirs: Set; + pluginFiles: Map; mcpServers: Record | undefined; }; @@ -105,6 +106,9 @@ export type PluginTargetDefinition = { /** Component directories copied into a plugin when it has no `components` override. */ defaultComponents: readonly string[]; + /** Build warnings for source content this layout omits by default. */ + omittedContentWarnings?: Record; + resolvePluginPath: ( pluginName: string, pluginConfig: EmittedPluginConfig, @@ -183,6 +187,7 @@ export type PluginTargetDefinition = { validateOutput: ( root: string, issues: ValidationIssue[], + managedPaths?: string[], ) => Promise; installSnippet: InstallSnippetDefinition; diff --git a/tests/target-preflight.test.ts b/tests/target-preflight.test.ts new file mode 100644 index 0000000..7f5d8dc --- /dev/null +++ b/tests/target-preflight.test.ts @@ -0,0 +1,234 @@ +import { readFile, writeFile, mkdir, access } from "node:fs/promises"; +import path from "node:path"; +import { Project, type ProjectArgs } from "fixturify-project"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { build } from "../src/build.js"; +import { validateOutput } from "../src/adapters.js"; +import type { TargetName } from "../src/types.js"; + +type DirJSON = NonNullable; +let project: Project | undefined; +afterEach(async () => { + vi.restoreAllMocks(); + await project?.dispose(); + project = undefined; +}); + +const SKILL = + "---\nname: search\ndescription: Search documents.\n---\n# Search\n"; +const ENTRY = + 'entry: { policy: { installation: "AVAILABLE", authentication: "ON_INSTALL" }, category: "Productivity" }'; + +async function setup(targets: string, source: DirJSON = {}): Promise { + project = new Project("target-preflight", "1.0.0", { + files: { + "pluginpack.config.ts": `export default { + name: "acme-plugins", version: "1.0.0", + metadata: { description: "Acme plugins.", author: { name: "Acme" } }, + targets: { ${targets} } + };`, + shared: { + acme: { skills: { search: { "SKILL.md": SKILL } }, ...source }, + }, + }, + }); + await project.write(); + return project.baseDir; +} + +describe("target build and validation agree", () => { + it("does not point Copilot at a nonexistent hooks/hooks.json", async () => { + const root = await setup( + 'copilot: { outDir: "out", plugins: { acme: { source: "shared/acme" } } }', + { + hooks: { "before-run.sh": "#!/bin/sh\necho ready\n" }, + }, + ); + await build({ cwd: root }); + const manifest = JSON.parse( + await readFile(path.join(root, "out/plugins/acme/plugin.json"), "utf8"), + ) as Record; + expect(manifest).not.toHaveProperty("hooks"); + await expect( + validateOutput("copilot", path.join(root, "out")), + ).resolves.toEqual({ ok: true, issues: [] }); + }); + + it.each(["claude", "cursor"])( + "validates %s output with a custom marketplaceDir", + async (target) => { + const root = await setup( + `${target}: { outDir: "out", marketplaceDir: "catalog/native", plugins: { acme: { source: "shared/acme" } } }`, + ); + await build({ cwd: root }); + await expect( + validateOutput(target, path.join(root, "out")), + ).resolves.toEqual({ ok: true, issues: [] }); + }, + ); + + it.each(["claude", "cursor"] as const)( + "uses the managed %s marketplace pair rather than an unrelated root file", + async (target) => { + const defaultDir = + target === "claude" ? ".claude-plugin" : ".cursor-plugin"; + const root = await setup( + `${target}: { outDir: "out", marketplaceDir: "catalog/native", rootFiles: { "${defaultDir}/marketplace.json": "unrelated.json" }, plugins: { acme: { source: "shared/acme" } } }`, + ); + await writeFile(path.join(root, "unrelated.json"), "{}\n"); + await build({ cwd: root }); + await expect( + validateOutput(target, path.join(root, "out")), + ).resolves.toEqual({ ok: true, issues: [] }); + }, + ); + + it.each([ + "claude", + "cursor", + "copilot", + "antigravity", + "codex", + "agent-plugins", + ])( + "rejects two %s plugins writing the same output path before writing anything", + async (target) => { + const root = + await setup(`${JSON.stringify(target)}: { outDir: "out", plugins: { + first: { source: "shared/acme", path: "plugins/common", ${target === "codex" ? ENTRY : ""} }, + second: { source: "shared/acme", path: "plugins/common", ${target === "codex" ? ENTRY : ""} } + } }`); + await expect(build({ cwd: root })).rejects.toThrow( + /first.*second.*plugins\/common/, + ); + await expect(access(path.join(root, "out"))).rejects.toThrow(); + await expect( + readFile(path.join(root, "shared/acme/skills/search/SKILL.md"), "utf8"), + ).resolves.toBe(SKILL); + }, + ); + + it("rejects a plugin file colliding with another plugin's generated manifest", async () => { + const root = await setup( + 'claude: { outDir: "out", plugins: { first: { source: "shared/acme", path: "common" }, second: { source: "shared/acme", path: "common/nested" } } }', + { + "plugin.pluginpack.json": JSON.stringify({ + additionalFiles: { + "nested/.claude-plugin/plugin.json": "extra.json", + }, + }), + "extra.json": "{}\n", + }, + ); + await expect(build({ cwd: root })).rejects.toThrow( + /first.*second.*common\/nested\/\.claude-plugin\/plugin.json/, + ); + }); + + it("does not let a source file overwrite a generated marketplace", async () => { + const root = await setup( + 'claude: { outDir: "out", plugins: { acme: { source: "shared/acme", path: "." } } }', + { + "plugin.pluginpack.json": JSON.stringify({ + additionalFiles: { ".claude-plugin/marketplace.json": "extra.json" }, + }), + "extra.json": "{}\n", + }, + ); + await expect(build({ cwd: root })).rejects.toThrow( + /marketplace.json.*collid|collid.*marketplace.json/, + ); + }); + + it.each(["codex", "agent-plugins"] as const)( + "reports missing Codex entry fields during %s build", + async (target) => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const root = await setup( + `${JSON.stringify(target)}: { outDir: "out", ${target === "agent-plugins" ? "clients: { codex: {} }," : ""} plugins: { acme: { source: "shared/acme" } } }`, + ); + await build({ cwd: root }); + const messages = warn.mock.calls.flat().join("\n"); + expect(messages).toContain("pluginpack.config.ts"); + expect(messages).toContain("policy.installation"); + expect(messages).toContain("policy.authentication"); + expect(messages).toContain("category"); + expect(messages).toContain( + target === "codex" + ? "plugins.acme.entry" + : "plugins.acme.clients.codex.entry", + ); + }, + ); + + it.each(["codex", "agent-plugins"] as const)( + "checks snake_case skill names during %s build", + async (target) => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const root = await setup( + `${JSON.stringify(target)}: { outDir: "out", plugins: { acme: { source: "shared/acme", ${target === "codex" ? ENTRY : ""} } } }`, + { + skills: { + snake_case: { + "SKILL.md": SKILL.replace("name: search", "name: snake_case"), + }, + }, + }, + ); + await build({ cwd: root }); + expect(warn.mock.calls.flat().join("\n")).toMatch( + /skills\/snake_case\/SKILL.md.*name.*hyphens/, + ); + }, + ); + + it("does not warn when merged Codex entry fields are supplied", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const root = await setup( + `"agent-plugins": { outDir: "out", clients: { codex: {} }, plugins: { acme: { source: "shared/acme", clients: { codex: { ${ENTRY} } } } } }`, + ); + await build({ cwd: root }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("reports hooks omitted when moving to agent-plugins with clients.codex", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const root = await setup( + `"agent-plugins": { outDir: "out", clients: { codex: {} }, plugins: { acme: { source: "shared/acme", clients: { codex: { ${ENTRY} } } } } }`, + { + hooks: { "hooks.json": '{ "hooks": {} }\n' }, + }, + ); + await build({ cwd: root }); + expect(warn.mock.calls.flat().join("\n")).toMatch( + /shared\/acme\/hooks.*not.*ship|shared\/acme\/hooks.*omit/, + ); + await expect( + access(path.join(root, "out/plugins/acme/hooks/hooks.json")), + ).rejects.toThrow(); + }); + + it("does not warn when hooks are explicitly excluded", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const root = await setup( + `"agent-plugins": { outDir: "out", clients: { codex: {} }, plugins: { acme: { source: "shared/acme", exclude: ["hooks"], clients: { codex: { ${ENTRY} } } } } }`, + { hooks: { "hooks.json": '{ "hooks": {} }\n' } }, + ); + await build({ cwd: root }); + expect(warn).not.toHaveBeenCalled(); + }); + + it("reports omitted hooks in overrides too", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const root = await setup( + `"agent-plugins": { outDir: "out", clients: { codex: {} }, plugins: { acme: { source: "shared/acme", overrides: "overrides/acme", clients: { codex: { ${ENTRY} } } } } }`, + ); + await mkdir(path.join(root, "overrides/acme/hooks"), { recursive: true }); + await writeFile( + path.join(root, "overrides/acme/hooks/hooks.json"), + '{ "hooks": {} }\n', + ); + await build({ cwd: root }); + expect(warn.mock.calls.flat().join("\n")).toContain("overrides/acme/hooks"); + }); +}); From dbabb55532b270de0c3d2a633ec26baf2eb0ab83 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 21 Oct 2026 14:16:24 -0700 Subject: [PATCH 2/9] docs: make the quick start executable across targets Provide every authored file, use Copilot-compatible agent names, and supply Codex marketplace policy fields. Materialize the README blocks in CI and run the built CLI for every configured target. Clarify oracle strength, custom marketplace validation, and target migration cleanup. --- CLAUDE.md | 2 +- README.md | 81 ++++++++++++++++++---- snippets/readme/quickstart-agent.md | 6 ++ snippets/readme/quickstart-run.bash | 6 ++ snippets/readme/quickstart-skill.md | 8 +++ snippets/readme/snippet-02.ts | 9 ++- src/index.ts | 2 +- tests/readme-quick-start.test.ts | 100 ++++++++++++++++++++++++++++ 8 files changed, 198 insertions(+), 16 deletions(-) create mode 100644 snippets/readme/quickstart-agent.md create mode 100644 snippets/readme/quickstart-run.bash create mode 100644 snippets/readme/quickstart-skill.md create mode 100644 tests/readme-quick-start.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 3afa8ae..0fc9f08 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,7 +34,7 @@ Data flows: **config → discover source → collect/render files → emit per t diff, prune, and validate all derive from it. - `src/cli.ts` — commander CLI: `init`, `build`, `validate`, `diff`, `prune`, - `clean`, `docs`. + `clean`, `install-info`, `docs`. - `src/schema.ts` — **zod schemas are the source of truth for config types**; the public types are derived with `z.infer`. Edit schemas here, not `types.ts`. - `src/types.ts` — non-config types; re-exports the config types from `schema.ts`. diff --git a/README.md b/README.md index 626eded..ed0fea7 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ One source of truth for agent plugins across AI app ecosystems. `pluginpack` is a build tool for compiling portable skills, commands, agents, rules, hooks, assets, and metadata into the native plugin layouts expected by each AI app. It copies files, writes target manifests, and validates generated output; it is not a package manager or publisher.

- pluginpack packages source components into Claude, Cursor, Antigravity, and Copilot plugin targets + pluginpack packages source components into Claude, Cursor, Antigravity, Copilot, Codex, and Agent Plugins targets

## Quick Start @@ -30,16 +30,45 @@ shared/ release-notes/ SKILL.md agents/ - search-assistant.md - assets/ - icon.png + search-assistant.agent.md pluginpack.config.ts ``` +Create `shared/acme/skills/release-notes/SKILL.md`: + + + +```md snippet=readme/quickstart-skill.md +--- +name: release-notes +description: Draft release notes from a list of changes. +--- + +# Release notes + +Group the changes by feature and fix. Write a short note for each group. +``` + +Create `shared/acme/agents/search-assistant.agent.md`. Copilot requires the +`NAME.agent.md` filename; the other targets accept it too: + + + +```md snippet=readme/quickstart-agent.md +--- +name: search-assistant +description: Find source documents that answer a question. +--- + +Search for relevant documents and cite the sources in your answer. +``` + Map that shared source directly into each native plugin output. Add `overrides` only when a target needs files that differ from or do not exist in the shared source. + + ```ts snippet=readme/snippet-02.ts import { defineConfig } from "@gleanwork/pluginpack"; @@ -57,7 +86,6 @@ export default defineConfig({ plugins: { acme: { source: "shared/acme", - overrides: "overrides/cursor/acme", path: "plugins/cursor/acme", }, }, @@ -84,18 +112,28 @@ export default defineConfig({ codex: { outDir: "plugins/codex", plugins: { - acme: { source: "shared/acme" }, + acme: { + source: "shared/acme", + entry: { + policy: { installation: "AVAILABLE", authentication: "ON_INSTALL" }, + category: "Productivity", + }, + }, }, }, }, }); ``` -Build and validate the generated outputs: +Build and validate every configured target: -```bash +```bash snippet=readme/quickstart-run.bash npx pluginpack build npx pluginpack validate --target cursor +npx pluginpack validate --target claude +npx pluginpack validate --target antigravity +npx pluginpack validate --target copilot +npx pluginpack validate --target codex ``` Claude, Cursor, Antigravity, Copilot, and Codex users install from the generated @@ -258,7 +296,9 @@ Each target compiles the same source into one app's native plugin layout: > **Heads up:** targets that write the same marketplace file need distinct `outDir`s (or separate repos): `claude` and `copilot` both write `.claude-plugin/marketplace.json`; `agent-plugins` with `clients.codex` writes the `codex` target's `.agents/plugins/marketplace.json`; and with `clients.copilot` it writes the `copilot` target's `.github/plugin/marketplace.json`. `build` errors on overlapping output paths. -New targets are added from official docs or real plugin examples — not guessed abstractions. +Oracle strength differs by target. See [the conformance oracles](./CONFORMANCE.md#oracles-the-harness-uses): Claude uses its CLI validator, Cursor uses Glean-authored schemas, and Codex and Agent Plugins use official package schemas. Codex and Copilot marketplace checks and Antigravity layout checks use structural assertions against product docs or published examples. + +New targets are added from official docs or real plugin examples, not guessed abstractions. ## Agent Plugins @@ -338,7 +378,9 @@ targets: { - A client-only content kind is selected by default, and allowed in `include`, only when pluginpack places it for a listed client. Today that is `agents` for `copilot`. pluginpack doesn't place `commands`, `rules`, or - `hooks` for any client yet. + `hooks` for any client yet. With `clients.codex`, build warns when it omits + source hooks. Keep the `codex` target if hooks are required, or explicitly + exclude `hooks` to acknowledge the omission. - Copilot's marketplace requires an owner, from `metadata.owner` or `metadata.author`. - `validate` checks every package, then each client marketplace that's @@ -350,6 +392,16 @@ targets: { valid plugin name, `mcp.json` servers, and skill names that follow the [Agent Skills specification](https://agentskills.io/specification) and match their directory. Conforming clients skip a skill that fails these checks. +Build warns about invalid skill frontmatter and missing Codex entry fields; +`validate` reports them as errors. Supply `policy.installation`, +`policy.authentication`, and `category` in `entry` (or `clients.codex.entry`). + +**Moving between `codex` and `agent-plugins` with `clients.codex`:** run +`pluginpack clean --target codex` while the old target is still configured, +before removing it or building its replacement into the same output directory. +This removes the old managed manifest and marketplace without deleting source +files. In the reverse direction, run `pluginpack clean --target agent-plugins` +before removing that target. The `agent-plugins` target does not ship Codex hooks. **Upgrading from 0.11:** the `codex` target's output layout changes. Run `pluginpack build`, and either publish the new layout or set @@ -500,7 +552,7 @@ Legacy 0.10 sources may still declare `additionalFiles` in } ``` -The map is destination (emitted plugin root relative) -> source (source plugin relative). Files are emitted verbatim at the plugin root, are tracked as managed output, and support target overrides: a `targets//` file wins for that host. A destination that collides with a component or static file is an error, and `pluginpack` does not build bundles — produce build output before running `pluginpack build` so the declared source paths exist. +The map is destination (emitted plugin root relative) -> source (source plugin relative). Files are emitted at the plugin root after partial substitution for supported text types (see [Partials](#partials)), are tracked as managed output, and support target overrides: a `targets//` file wins for that host. A destination that collides with a component or static file is an error, and `pluginpack` does not build bundles — produce build output before running `pluginpack build` so the declared source paths exist. ## Partials @@ -609,7 +661,7 @@ rest of the artifact. | `outDir` | string | yes | Output directory for this target, relative to the config root. | | `format` | `"agent-plugins"` \| `"legacy"` | no | `codex` only. `"legacy"` keeps the `.codex-plugin/plugin.json` layout; default `"agent-plugins"`. | | `plugins` | record | yes | Emitted plugins, keyed by emitted plugin name (see **`targets..plugins.`**). | -| `marketplaceDir` | string (safe relative) | no | Override the marketplace dir (defaults: `.cursor-plugin` / `.claude-plugin`). | +| `marketplaceDir` | string (safe relative) | no | Override the marketplace and plugin-manifest dir for `cursor` / `claude` (defaults: `.cursor-plugin` / `.claude-plugin`). | | `pluginRoot` | string (safe relative) | no | Override the plugin root dir (`claude`; defaults to `plugins`). | | `version` | string | no | Override the version for this target (defaults to top-level `version`). | | `manifest` | object | no | Deep-merged into the generated marketplace manifest. An error on `agent-plugins`; use `clients..manifest`. | @@ -620,6 +672,11 @@ rest of the artifact. | `updateCheck` | `{ repository? }` | no | Generate a session-start update-check hook (`claude`/`cursor` only; see **Update Check**). | | `repository` | string | no | Repo this target's output lives in, for `install-info` (defaults to `metadata.repository`). | +With a custom `marketplaceDir`, retain `.pluginpack/` alongside the output. +Validation uses that target's managed paths to locate both the marketplace and +plugin manifests without scanning unrelated repository files. Without a managed +manifest, validation uses the target's standard directory. + **`targets..plugins.`** | Field | Type | Required | Meaning | diff --git a/snippets/readme/quickstart-agent.md b/snippets/readme/quickstart-agent.md new file mode 100644 index 0000000..6b2f564 --- /dev/null +++ b/snippets/readme/quickstart-agent.md @@ -0,0 +1,6 @@ +--- +name: search-assistant +description: Find source documents that answer a question. +--- + +Search for relevant documents and cite the sources in your answer. diff --git a/snippets/readme/quickstart-run.bash b/snippets/readme/quickstart-run.bash new file mode 100644 index 0000000..7af4896 --- /dev/null +++ b/snippets/readme/quickstart-run.bash @@ -0,0 +1,6 @@ +npx pluginpack build +npx pluginpack validate --target cursor +npx pluginpack validate --target claude +npx pluginpack validate --target antigravity +npx pluginpack validate --target copilot +npx pluginpack validate --target codex diff --git a/snippets/readme/quickstart-skill.md b/snippets/readme/quickstart-skill.md new file mode 100644 index 0000000..99b54be --- /dev/null +++ b/snippets/readme/quickstart-skill.md @@ -0,0 +1,8 @@ +--- +name: release-notes +description: Draft release notes from a list of changes. +--- + +# Release notes + +Group the changes by feature and fix. Write a short note for each group. diff --git a/snippets/readme/snippet-02.ts b/snippets/readme/snippet-02.ts index d9d44ee..582a2fa 100644 --- a/snippets/readme/snippet-02.ts +++ b/snippets/readme/snippet-02.ts @@ -14,7 +14,6 @@ export default defineConfig({ plugins: { acme: { source: "shared/acme", - overrides: "overrides/cursor/acme", path: "plugins/cursor/acme", }, }, @@ -41,7 +40,13 @@ export default defineConfig({ codex: { outDir: "plugins/codex", plugins: { - acme: { source: "shared/acme" }, + acme: { + source: "shared/acme", + entry: { + policy: { installation: "AVAILABLE", authentication: "ON_INSTALL" }, + category: "Productivity", + }, + }, }, }, }, diff --git a/src/index.ts b/src/index.ts index 9c51b51..575e34b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ -/** Public API — everything the CLI itself uses is exported here for programmatic use. */ +/** Public build, validation, cleanup, config, and install-snippet APIs for programmatic use. */ export { defineConfig, loadConfig } from "./config.js"; export { build } from "./build.js"; export { clean, prune } from "./cleanup.js"; diff --git a/tests/readme-quick-start.test.ts b/tests/readme-quick-start.test.ts new file mode 100644 index 0000000..298e55f --- /dev/null +++ b/tests/readme-quick-start.test.ts @@ -0,0 +1,100 @@ +import { execFileSync } from "node:child_process"; +import { + mkdtemp, + mkdir, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach, expect, it } from "vitest"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +let fixture: string | undefined; +afterEach(async () => { + if (fixture) await rm(fixture, { recursive: true, force: true }); +}); + +it("materializes the README Quick Start and runs build and validate for every configured target", async () => { + const readme = await readFile(path.join(root, "README.md"), "utf8"); + const quickStart = readme + .split("## Quick Start\n")[1] + ?.split("\n## Mental Model")[0]; + expect(quickStart).toBeDefined(); + const files = new Map(); + for (const match of quickStart!.matchAll( + /\s*```[^\n]*\n([\s\S]*?)\n```/g, + )) { + files.set(match[1], `${match[2]}\n`); + } + expect(files.has("pluginpack.config.ts")).toBe(true); + + // The tree must match the files shown, so adding a required path to either + // the tree or the config without updating the example fails the test. + const tree = /```tree\n([\s\S]*?)\n```/.exec(quickStart!)?.[1]; + expect(tree).toBeDefined(); + const parents: string[] = []; + const treeFiles: string[] = []; + for (const line of tree!.split("\n")) { + const name = line.trim(); + const depth = (line.length - line.trimStart().length) / 2; + parents.length = depth; + if (name.endsWith("/")) parents.push(name.slice(0, -1)); + else treeFiles.push([...parents, name].join("/")); + } + expect([...files.keys()].sort()).toEqual(treeFiles.sort()); + + fixture = await mkdtemp(path.join(os.tmpdir(), "pluginpack-readme-")); + for (const [file, contents] of files) { + await mkdir(path.dirname(path.join(fixture, file)), { recursive: true }); + await writeFile(path.join(fixture, file), contents); + } + // Use the real local package and its built CLI, not the network or tsx. + // The config keeps its literal public package import from the README. + await mkdir(path.join(fixture, "node_modules/@gleanwork"), { + recursive: true, + }); + await symlink( + root, + path.join(fixture, "node_modules/@gleanwork/pluginpack"), + "dir", + ); + await writeFile(path.join(fixture, "package.json"), '{ "type": "module" }\n'); + const commands = + /```bash snippet=readme\/quickstart-run.bash\n([\s\S]*?)\n```/.exec( + quickStart!, + )?.[1]; + expect(commands).toBeDefined(); + const validated: string[] = []; + for (const command of commands!.split("\n")) { + expect(command).toMatch(/^npx pluginpack /); + const args = command.slice("npx pluginpack ".length).split(" "); + execFileSync(process.execPath, [path.join(root, "dist/cli.js"), ...args], { + cwd: fixture, + encoding: "utf8", + stdio: "pipe", + }); + if (args[0] === "validate") + validated.push(args[args.indexOf("--target") + 1]); + } + const { loadConfig } = await import("../src/config.js"); + const project = await loadConfig(fixture); + expect(validated.sort()).toEqual(Object.keys(project.config.targets).sort()); +}); + +it("documents oracle strength and the current CLI and target names", async () => { + const readme = await readFile(path.join(root, "README.md"), "utf8"); + const targets = readme + .split("## Targets\n")[1] + ?.split("\n## Agent Plugins")[0]; + expect(targets).toContain("CONFORMANCE.md#oracles-the-harness-uses"); + expect(readme).toMatch(/alt="[^"]*Codex[^"]*Agent Plugins/); + expect(readme).not.toContain("Files are emitted verbatim at the plugin root"); + expect(readme).toMatch(/clean --target codex`[\s\S]*?before removing it/); + expect(readme).toContain("clean --target agent-plugins"); + const guide = await readFile(path.join(root, "CLAUDE.md"), "utf8"); + expect(guide).toMatch(/commander CLI:[\s\S]*?`install-info`/); +}); From 3fb6d683a047987ebe7c7fc8f58f855c9758dc6b Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 21 Oct 2026 14:16:53 -0700 Subject: [PATCH 3/9] test(conformance): enforce a 90-day evidence refresh cadence Fail CI when citation or vendored-schema verification dates expire, are invalid, or are in the future. Preserve the recorded verification dates and document how to recheck claims and diff pinned schemas before advancing them. --- CONFORMANCE.md | 34 ++++++++++-- tests/evidence-freshness.test.ts | 76 ++++++++++++++++++++++++++ tests/fixtures/agent-plugins/SOURCE.md | 6 ++ tests/fixtures/cursor/SOURCE.md | 10 +++- 4 files changed, 117 insertions(+), 9 deletions(-) create mode 100644 tests/evidence-freshness.test.ts diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 06375bf..46270d6 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -24,8 +24,9 @@ against a temp fixture via [`bintastic`](https://github.com/scalvert/bintastic). - **claude** — runs `claude plugin validate --strict` against the emitted marketplace and plugin directories when the `claude` CLI is on `PATH` (skipped - otherwise, e.g. CI). This is Anthropic's own validator — the same check their - submission pipeline runs — so it is a genuinely external oracle that tracks + otherwise). CI installs the CLI before running the suite. This is Anthropic's + own validator, the same check their submission pipeline runs, so it is an + external oracle that tracks upstream automatically. A structural golden assertion also pins the exact emitted shape. - **cursor** — validates the emitted `marketplace.json` and `plugin.json` against @@ -75,10 +76,12 @@ against a temp fixture via [`bintastic`](https://github.com/scalvert/bintastic). `name` is required and `skills`/`hooks`/`mcpServers` are optional pointers; `validate` detects either layout per plugin. Every marketplace entry must carry `policy.installation`, `policy.authentication`, and `category`; pluginpack has - no way to infer these, so the base entry stays guess-free and `validateOutput` - errors clearly if an author never supplies them via the per-plugin `entry` - passthrough. An entry's `source` is a bare string only for local plugins (the - only shape pluginpack itself ever emits); a `url`/`git-subdir`/`npm` source + no way to infer these, so the base entry stays guess-free. Build warns and + `validateOutput` errors if an author never supplies them via the per-plugin + `entry` passthrough (or `clients.codex.entry` for a package client). Build + also warns about skill frontmatter that package validation rejects. + An entry's `source` is a bare string only for local plugins (the only shape + pluginpack itself ever emits); a `url`/`git-subdir`/`npm` source added via `entry` is a structured object with an inner `source` discriminator (e.g. `{ source: "git-subdir", url, path, ref }`), validated by shape rather than requiring a local directory to exist. No published JSON Schema exists. @@ -144,6 +147,25 @@ repository URL: , verified 2026-10-02). +## Evidence refresh cadence + +Re-verify every citation and vendored schema at least every 90 days. +`tests/evidence-freshness.test.ts` runs in `npm test` and CI. It checks every +literal `verifiedAt` in `src/` and the `Verified-at` date in each vendored +schema directory's `SOURCE.md`. It fails when a date is more than 90 days old, +is in the future, or is invalid. Tests do not fetch the network. + +To refresh a citation, open its `documentationUrl`, check the claim against the +current product docs, and run the affected target's conformance tests. Update +`verifiedAt` only after that check. Correct the implementation or record a +format conflict here if the upstream behavior changed. + +To refresh a vendored schema, follow the fetch commands in its `SOURCE.md`, +diff the fetched bytes against the pinned copy, and run +`npx vitest run tests/conformance.test.ts tests/evidence-freshness.test.ts`. +Record the checked commit and update `Verified-at: YYYY-MM-DD` in `SOURCE.md`, +even when the bytes are unchanged. Do not advance dates just to pass CI. + ## Refreshing vendored schemas The Cursor and Agent Plugins schemas are pinned copies. To update them, re-fetch diff --git a/tests/evidence-freshness.test.ts b/tests/evidence-freshness.test.ts new file mode 100644 index 0000000..2454851 --- /dev/null +++ b/tests/evidence-freshness.test.ts @@ -0,0 +1,76 @@ +import { readFile, readdir } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +const CADENCE_DAYS = 90; +const DAY = 24 * 60 * 60 * 1000; +const REFRESH = + "Re-open the cited docs or re-fetch and diff the schemas, verify the claims, then update verifiedAt / Verified-at. See CONFORMANCE.md#evidence-refresh-cadence."; + +function freshnessProblem(date: string, today: number): string | undefined { + const verified = Date.parse(`${date}T00:00:00Z`); + if ( + !/^\d{4}-\d{2}-\d{2}$/.test(date) || + !Number.isFinite(verified) || + new Date(verified).toISOString().slice(0, 10) !== date + ) { + return `invalid verification date "${date}"`; + } + const age = Math.floor(today / DAY) - Math.floor(verified / DAY); + if (age < 0) return `verification date ${date} is in the future`; + if (age > CADENCE_DAYS) + return `verification date ${date} is ${age} days old (cadence: ${CADENCE_DAYS} days)`; + return undefined; +} + +it("rejects stale, future, and invalid evidence dates at the 90-day boundary", () => { + const today = Date.parse("2026-10-21T12:00:00Z"); + expect(freshnessProblem("2026-07-23", today)).toBeUndefined(); + expect(freshnessProblem("2026-07-22", today)).toContain("91 days old"); + expect(freshnessProblem("2026-10-22", today)).toContain("future"); + expect(freshnessProblem("2026-02-30", today)).toContain("invalid"); +}); + +it("keeps every source citation verifiedAt within the documented cadence", async () => { + const problems: string[] = []; + let citations = 0; + for (const file of await readdir(path.join(root, "src"), { + recursive: true, + })) { + if (!file.endsWith(".ts")) continue; + const contents = await readFile(path.join(root, "src", file), "utf8"); + for (const match of contents.matchAll(/verifiedAt:\s*"([^"]+)"/g)) { + citations++; + const problem = freshnessProblem(match[1], Date.now()); + if (problem) problems.push(`src/${file}: ${problem}. ${REFRESH}`); + } + } + expect(citations).toBeGreaterThan(0); + expect(problems).toEqual([]); +}); + +it("keeps each vendored schema's SOURCE.md verification date within the same cadence", async () => { + const problems: string[] = []; + for (const format of ["cursor", "agent-plugins"]) { + const source = `tests/fixtures/${format}/SOURCE.md`; + const contents = await readFile(path.join(root, source), "utf8"); + const date = /^- Verified-at: (\S+)$/m.exec(contents)?.[1]; + const schemas = ( + await readdir(path.join(root, "tests/fixtures", format)) + ).filter((file) => file.endsWith(".schema.json")); + expect(schemas.length).toBeGreaterThan(0); + for (const schema of schemas) { + const problem = date + ? freshnessProblem(date, Date.now()) + : "missing Verified-at: YYYY-MM-DD in SOURCE.md"; + if (problem) + problems.push(`${source} (${schema}): ${problem}. ${REFRESH}`); + } + } + expect(problems).toEqual([]); + expect(await readFile(path.join(root, "CONFORMANCE.md"), "utf8")).toContain( + `Re-verify every citation and vendored schema at least every ${CADENCE_DAYS} days.`, + ); +}); diff --git a/tests/fixtures/agent-plugins/SOURCE.md b/tests/fixtures/agent-plugins/SOURCE.md index 3bf8d78..73ab442 100644 --- a/tests/fixtures/agent-plugins/SOURCE.md +++ b/tests/fixtures/agent-plugins/SOURCE.md @@ -7,6 +7,7 @@ TSC (not by this repo). - Source: `agentplugins/agent-plugins-spec` → `schemas/1.0.0/` - Commit: `ff8ab5e392cc87bd88d87c060815a87490e51003` (fetched 2026-10-02) +- Verified-at: 2026-10-02 - SHA-256: - `plugin.schema.json` — `0a4aad95ce337878ad38802ebf0daa3fde76abe3f65400c86bcbb1ec0b3ab883` - `mcp.schema.json` — `6539175bfcdf43085855183e86da40ea94b166547a72b47ae9a0a390516d3acb` @@ -16,6 +17,11 @@ The specification text is authoritative where it and these schemas disagree different contents (spec §10.1), so the 1.0.0 copies shouldn't change. A new specification version publishes new schemas under a new directory. +Re-verify at least every 90 days. `tests/evidence-freshness.test.ts` enforces +this cadence in CI. Re-fetch and diff the bytes, then update `Verified-at` and +record the checked upstream commit, even when the schemas are unchanged. See +[the refresh procedure](../../../CONFORMANCE.md#evidence-refresh-cadence). + Do not hand-edit. Re-fetch to update: ```bash diff --git a/tests/fixtures/cursor/SOURCE.md b/tests/fixtures/cursor/SOURCE.md index fe54d83..957f08f 100644 --- a/tests/fixtures/cursor/SOURCE.md +++ b/tests/fixtures/cursor/SOURCE.md @@ -5,6 +5,7 @@ plugin/marketplace JSON Schemas, authored outside this repo. - Source: `gleanwork/cursor-plugins` → `schemas/` - Commit: `424c3485` (fetched 2026-05-29) +- Verified-at: 2026-07-28 - Re-verified byte-identical against upstream `schemas/` (commit `6f3ad6e6`) on 2026-07-28 — no re-fetch needed. @@ -17,6 +18,9 @@ gh api repos/gleanwork/cursor-plugins/contents/schemas/plugin.schema.json \ -H "Accept: application/vnd.github.raw" > plugin.schema.json ``` -Staleness has no automated check today — re-run the fetch above and diff -against the vendored copies periodically, especially before a stability-tier -bump, since CONFORMANCE.md already flags this oracle as self-referential. +Re-verify at least every 90 days. `tests/evidence-freshness.test.ts` enforces +this cadence in CI. Run the fetch above and diff against the vendored copies, +then update `Verified-at` and record the checked upstream commit. See +[the refresh procedure](../../../CONFORMANCE.md#evidence-refresh-cadence). +CONFORMANCE.md flags this oracle as self-referential because Glean, not Cursor, +authored these schemas. From 1029200b0d65fd4c857fd4bfde384132d59aa8c2 Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 21 Oct 2026 14:41:04 -0700 Subject: [PATCH 4/9] test(conformance): check evidence dates on a schedule, not on every PR The freshness checks depend on the date, not on the change under test, so running them in npm test would turn every pull request red the day a citation ages out. Run them weekly and on demand instead, and keep the pure boundary test in the suite. --- .github/workflows/evidence-freshness.yml | 29 +++++++++ CONFORMANCE.md | 14 ++-- tests/evidence-freshness.test.ts | 82 ++++++++++++++---------- 3 files changed, 86 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/evidence-freshness.yml diff --git a/.github/workflows/evidence-freshness.yml b/.github/workflows/evidence-freshness.yml new file mode 100644 index 0000000..497ec79 --- /dev/null +++ b/.github/workflows/evidence-freshness.yml @@ -0,0 +1,29 @@ +name: Evidence freshness + +# Fails when a target citation or vendored schema is past the re-verification +# cadence in CONFORMANCE.md. It runs on a schedule rather than on pull +# requests, because its result depends on the date, not on the change. +on: + schedule: + - cron: "0 16 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + freshness: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Set up toolchain + uses: jdx/mise-action@v4 + + - name: Install dependencies + run: npm ci + + - name: Check evidence dates + env: + PLUGINPACK_CHECK_FRESHNESS: "1" + run: npx vitest run tests/evidence-freshness.test.ts diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 46270d6..b8b8116 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -150,10 +150,14 @@ verified 2026-10-02). ## Evidence refresh cadence Re-verify every citation and vendored schema at least every 90 days. -`tests/evidence-freshness.test.ts` runs in `npm test` and CI. It checks every -literal `verifiedAt` in `src/` and the `Verified-at` date in each vendored -schema directory's `SOURCE.md`. It fails when a date is more than 90 days old, -is in the future, or is invalid. Tests do not fetch the network. +The `Evidence freshness` workflow runs `tests/evidence-freshness.test.ts` every +Monday and on demand. It checks every literal `verifiedAt` in `src/` and the +`Verified-at` date in each vendored schema directory's `SOURCE.md`. It fails +when a date is more than 90 days old, is in the future, or is invalid. The date +checks do not run in `npm test`, because their result depends on the day rather +than the change. Run them locally with +`env PLUGINPACK_CHECK_FRESHNESS=1 npx vitest run tests/evidence-freshness.test.ts`. +The tests do not fetch the network. To refresh a citation, open its `documentationUrl`, check the claim against the current product docs, and run the affected target's conformance tests. Update @@ -162,7 +166,7 @@ format conflict here if the upstream behavior changed. To refresh a vendored schema, follow the fetch commands in its `SOURCE.md`, diff the fetched bytes against the pinned copy, and run -`npx vitest run tests/conformance.test.ts tests/evidence-freshness.test.ts`. +`npx vitest run tests/conformance.test.ts` and the freshness command above. Record the checked commit and update `Verified-at: YYYY-MM-DD` in `SOURCE.md`, even when the bytes are unchanged. Do not advance dates just to pass CI. diff --git a/tests/evidence-freshness.test.ts b/tests/evidence-freshness.test.ts index 2454851..eb9ad42 100644 --- a/tests/evidence-freshness.test.ts +++ b/tests/evidence-freshness.test.ts @@ -4,6 +4,11 @@ import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; const root = fileURLToPath(new URL("../", import.meta.url)); +// The date checks below depend on the wall clock, so they would turn every +// PR red the day a citation ages out, whatever the PR changed. They run on a +// schedule instead (.github/workflows/evidence-freshness.yml) and on demand +// with PLUGINPACK_CHECK_FRESHNESS=1. +const checkFreshness = process.env.PLUGINPACK_CHECK_FRESHNESS === "1"; const CADENCE_DAYS = 90; const DAY = 24 * 60 * 60 * 1000; const REFRESH = @@ -33,43 +38,52 @@ it("rejects stale, future, and invalid evidence dates at the 90-day boundary", ( expect(freshnessProblem("2026-02-30", today)).toContain("invalid"); }); -it("keeps every source citation verifiedAt within the documented cadence", async () => { - const problems: string[] = []; - let citations = 0; - for (const file of await readdir(path.join(root, "src"), { - recursive: true, - })) { - if (!file.endsWith(".ts")) continue; - const contents = await readFile(path.join(root, "src", file), "utf8"); - for (const match of contents.matchAll(/verifiedAt:\s*"([^"]+)"/g)) { - citations++; - const problem = freshnessProblem(match[1], Date.now()); - if (problem) problems.push(`src/${file}: ${problem}. ${REFRESH}`); +it.runIf(checkFreshness)( + "keeps every source citation verifiedAt within the documented cadence", + async () => { + const problems: string[] = []; + let citations = 0; + for (const file of await readdir(path.join(root, "src"), { + recursive: true, + })) { + if (!file.endsWith(".ts")) continue; + const contents = await readFile(path.join(root, "src", file), "utf8"); + for (const match of contents.matchAll(/verifiedAt:\s*"([^"]+)"/g)) { + citations++; + const problem = freshnessProblem(match[1], Date.now()); + if (problem) problems.push(`src/${file}: ${problem}. ${REFRESH}`); + } } - } - expect(citations).toBeGreaterThan(0); - expect(problems).toEqual([]); -}); + expect(citations).toBeGreaterThan(0); + expect(problems).toEqual([]); + }, +); -it("keeps each vendored schema's SOURCE.md verification date within the same cadence", async () => { - const problems: string[] = []; - for (const format of ["cursor", "agent-plugins"]) { - const source = `tests/fixtures/${format}/SOURCE.md`; - const contents = await readFile(path.join(root, source), "utf8"); - const date = /^- Verified-at: (\S+)$/m.exec(contents)?.[1]; - const schemas = ( - await readdir(path.join(root, "tests/fixtures", format)) - ).filter((file) => file.endsWith(".schema.json")); - expect(schemas.length).toBeGreaterThan(0); - for (const schema of schemas) { - const problem = date - ? freshnessProblem(date, Date.now()) - : "missing Verified-at: YYYY-MM-DD in SOURCE.md"; - if (problem) - problems.push(`${source} (${schema}): ${problem}. ${REFRESH}`); +it.runIf(checkFreshness)( + "keeps each vendored schema's SOURCE.md verification date within the same cadence", + async () => { + const problems: string[] = []; + for (const format of ["cursor", "agent-plugins"]) { + const source = `tests/fixtures/${format}/SOURCE.md`; + const contents = await readFile(path.join(root, source), "utf8"); + const date = /^- Verified-at: (\S+)$/m.exec(contents)?.[1]; + const schemas = ( + await readdir(path.join(root, "tests/fixtures", format)) + ).filter((file) => file.endsWith(".schema.json")); + expect(schemas.length).toBeGreaterThan(0); + for (const schema of schemas) { + const problem = date + ? freshnessProblem(date, Date.now()) + : "missing Verified-at: YYYY-MM-DD in SOURCE.md"; + if (problem) + problems.push(`${source} (${schema}): ${problem}. ${REFRESH}`); + } } - } - expect(problems).toEqual([]); + expect(problems).toEqual([]); + }, +); + +it("documents the cadence the scheduled check enforces", async () => { expect(await readFile(path.join(root, "CONFORMANCE.md"), "utf8")).toContain( `Re-verify every citation and vendored schema at least every ${CADENCE_DAYS} days.`, ); From d0c90a67fa4117a1a3e6188c4da70b5374af1a0f Mon Sep 17 00:00:00 2001 From: Steve Calvert Date: Wed, 21 Oct 2026 15:41:25 -0700 Subject: [PATCH 5/9] fix(targets): enforce native layouts and clarify build diagnostics Claude Code and Cursor discover manifests at fixed paths, so reject custom marketplace directories before emission instead of validating an output the clients cannot load. Legacy Codex hooks pointers must name a file that actually ships. Point skill warnings at authored files, including overrides, and keep the target illustration, collision guidance, and executable quick start aligned with the output. Refresh only target citations confirmed against current sources. --- README.md | 44 ++++----- assets/pluginpack-flow.svg | 41 ++++++--- src/agent-plugins.ts | 14 ++- src/schema.ts | 37 ++++++++ src/targets/antigravity.ts | 6 +- src/targets/claude.ts | 33 +++---- src/targets/codex.ts | 5 +- src/targets/copilot.ts | 10 +-- src/targets/cursor.ts | 35 +++----- src/targets/engine.ts | 62 ++++++++++++- src/targets/shared.ts | 25 ------ src/targets/types.ts | 3 + tests/readme-quick-start.test.ts | 38 ++++++-- tests/target-preflight.test.ts | 149 +++++++++++++++++++++++++------ 14 files changed, 356 insertions(+), 146 deletions(-) diff --git a/README.md b/README.md index ed0fea7..9651df1 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ Each target compiles the same source into one app's native plugin layout: | `codex` | [OpenAI Codex / ChatGPT plugins](https://developers.openai.com/codex/plugins/build) | `.agents/plugins/marketplace.json`; an [Agent Plugins](#agent-plugins) package per plugin (`plugin.json` + optional `mcp.json`) | | `agent-plugins` | [Agent Plugins 1.0](https://agent-plugins.org/specification) | a portable package per plugin under `plugins//`; no marketplace unless `clients` is set ([one package for several clients](#one-package-for-several-clients)) | -> **Heads up:** targets that write the same marketplace file need distinct `outDir`s (or separate repos): `claude` and `copilot` both write `.claude-plugin/marketplace.json`; `agent-plugins` with `clients.codex` writes the `codex` target's `.agents/plugins/marketplace.json`; and with `clients.copilot` it writes the `copilot` target's `.github/plugin/marketplace.json`. `build` errors on overlapping output paths. +> **Heads up:** targets that write the same marketplace file need distinct `outDir`s (or separate repos): `claude` and `copilot` both write `.claude-plugin/marketplace.json`; `agent-plugins` with `clients.codex` writes the `codex` target's `.agents/plugins/marketplace.json`; and with `clients.copilot` it writes the `copilot` target's `.github/plugin/marketplace.json`. `build` errors on overlapping output paths. Collision checks normalize Unicode to NFC and ignore case on every filesystem: `assets/Logo.png` and `assets/logo.png` collide even on a case-sensitive volume. Rename one file so the output stays portable. Oracle strength differs by target. See [the conformance oracles](./CONFORMANCE.md#oracles-the-harness-uses): Claude uses its CLI validator, Cursor uses Glean-authored schemas, and Codex and Agent Plugins use official package schemas. Codex and Copilot marketplace checks and Antigravity layout checks use structural assertions against product docs or published examples. @@ -656,26 +656,28 @@ rest of the artifact. **`targets.`** — `` is one of `cursor`, `claude`, `antigravity`, `copilot`, `codex`, `agent-plugins`. -| Field | Type | Required | Meaning | -| ------------------ | ------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `outDir` | string | yes | Output directory for this target, relative to the config root. | -| `format` | `"agent-plugins"` \| `"legacy"` | no | `codex` only. `"legacy"` keeps the `.codex-plugin/plugin.json` layout; default `"agent-plugins"`. | -| `plugins` | record | yes | Emitted plugins, keyed by emitted plugin name (see **`targets..plugins.`**). | -| `marketplaceDir` | string (safe relative) | no | Override the marketplace and plugin-manifest dir for `cursor` / `claude` (defaults: `.cursor-plugin` / `.claude-plugin`). | -| `pluginRoot` | string (safe relative) | no | Override the plugin root dir (`claude`; defaults to `plugins`). | -| `version` | string | no | Override the version for this target (defaults to top-level `version`). | -| `manifest` | object | no | Deep-merged into the generated marketplace manifest. An error on `agent-plugins`; use `clients..manifest`. | -| `clients` | `{ codex?, copilot? }` | no | `agent-plugins` only. Clients each package serves; each gets a marketplace. `clients..manifest` is deep-merged into that client's marketplace. | -| `ignoredDiffPaths` | string[] | no | Output-relative paths `diff` ignores (a dir entry ignores everything below it). | -| `repositoryFiles` | string (safe relative) | no | Directory copied recursively into the generated repository root. | -| `rootFiles` | record (safe relative) | no | Legacy 0.10 output path → source path map; migrate to `repositoryFiles`. | -| `updateCheck` | `{ repository? }` | no | Generate a session-start update-check hook (`claude`/`cursor` only; see **Update Check**). | -| `repository` | string | no | Repo this target's output lives in, for `install-info` (defaults to `metadata.repository`). | - -With a custom `marketplaceDir`, retain `.pluginpack/` alongside the output. -Validation uses that target's managed paths to locate both the marketplace and -plugin manifests without scanning unrelated repository files. Without a managed -manifest, validation uses the target's standard directory. +| Field | Type | Required | Meaning | +| ------------------ | ------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `outDir` | string | yes | Output directory for this target, relative to the config root. | +| `format` | `"agent-plugins"` \| `"legacy"` | no | `codex` only. `"legacy"` keeps the `.codex-plugin/plugin.json` layout; default `"agent-plugins"`. | +| `plugins` | record | yes | Emitted plugins, keyed by emitted plugin name (see **`targets..plugins.`**). | +| `marketplaceDir` | string (safe relative) | no | Compatibility field for `cursor` / `claude`: only the fixed host directory (`.cursor-plugin` / `.claude-plugin`) is accepted. Other targets do not use it. | +| `pluginRoot` | string (safe relative) | no | Override the plugin root dir (`claude`; defaults to `plugins`). | +| `version` | string | no | Override the version for this target (defaults to top-level `version`). | +| `manifest` | object | no | Deep-merged into the generated marketplace manifest. An error on `agent-plugins`; use `clients..manifest`. | +| `clients` | `{ codex?, copilot? }` | no | `agent-plugins` only. Clients each package serves; each gets a marketplace. `clients..manifest` is deep-merged into that client's marketplace. | +| `ignoredDiffPaths` | string[] | no | Output-relative paths `diff` ignores (a dir entry ignores everything below it). | +| `repositoryFiles` | string (safe relative) | no | Directory copied recursively into the generated repository root. | +| `rootFiles` | record (safe relative) | no | Legacy 0.10 output path → source path map; migrate to `repositoryFiles`. | +| `updateCheck` | `{ repository? }` | no | Generate a session-start update-check hook (`claude`/`cursor` only; see **Update Check**). | +| `repository` | string | no | Repo this target's output lives in, for `install-info` (defaults to `metadata.repository`). | + +Claude Code and Cursor require their native manifest paths: `.claude-plugin/` +and `.cursor-plugin/`, respectively. Remove a custom `marketplaceDir`; use +`outDir` to move the generated repository instead. Claude's validator reports +`No manifest found in directory. Expected .claude-plugin/marketplace.json or .claude-plugin/plugin.json` +for a relocated manifest. [Cursor's reference](https://cursor.com/docs/reference/plugins) +says to place the marketplace at `.cursor-plugin/marketplace.json`. **`targets..plugins.`** diff --git a/assets/pluginpack-flow.svg b/assets/pluginpack-flow.svg index 1458ffe..cada094 100644 --- a/assets/pluginpack-flow.svg +++ b/assets/pluginpack-flow.svg @@ -9,7 +9,7 @@ pluginpack component packaging flow A source repository with shared components is built by pluginpack into native plugin targets for Claude, - Cursor, Antigravity, and Copilot. + Cursor, Antigravity, Copilot, Codex, and Agent Plugins.