diff --git a/extension-duplicate-check/README.md b/extension-duplicate-check/README.md index f5557ab..9ca9244 100644 --- a/extension-duplicate-check/README.md +++ b/extension-duplicate-check/README.md @@ -74,7 +74,7 @@ raycast/extensions doesn't run Dependabot, so bump the SHA by hand after changin | `scorer` | `lexical` | `lexical` scores in process. `jev` shortlists with `lexical` and asks TypeSafe's Jev model to judge each match. | | `typesafe-api-key` | | TypeSafe API key, required by the `jev` scorer. Pass it from a secret. | | `typesafe-model` | `jev-1.14.0` | TypeSafe model name or alias for the `jev` scorer. Pinned because thresholds are tuned per version and `jev-latest` can move. | -| `threshold` | `0.5` | Score (0–1) at or above which a match counts as a possible duplicate. | +| `threshold` | `0.25` | Score (0–1) at or above which a match counts as a possible duplicate. | | `min-score` | `0.2` | Matches below this score are left out of the comment and the `matches` output. | | `top-n` | `5` | Number of matches listed per source (store and open pull requests) for each new extension. | | `label` | | Label added when a possible duplicate is found and removed otherwise. Empty disables labeling. | @@ -131,7 +131,7 @@ The `lexical` scorer combines two signals: 0.8 count as unrelated. The score is `1 - (1 - content) × (1 - 0.5 × name)`, so either signal alone can raise it and identical names alone reach -0.5. The default threshold of 0.5 comes from a backtest over three months of new-extension pull requests: it flagged +0.5. A threshold of 0.5 in a backtest over three months of new-extension pull requests flagged 37% of those closed as duplicates of a store extension and 11% of those that were merged. The lexical scorer misses duplicates that describe the same thing in different words. In the same backtest it ranked @@ -168,6 +168,12 @@ over the same three months, with `jev-1.14.0`: Most of the merged extensions it flags overlap a lot with an existing one, such as WHost and iHosts, or Ghq Plus and ghq, so they are worth a reviewer's look anyway. Of the 17 duplicates it misses at 0.5, 12 score below 0.4. +The default threshold is 0.25, lower than anything in this backtest, so that matches like Everything Search, which +scored 0.40 against the Store's Everything, get flagged. Expect more flagged merged extensions than the table shows. + +Each run logs up to 10 matches per new extension as `Match: {...}` JSON lines, with `covers`, `services` and the other +signals, including matches below `min-score`. + The relation also words the comment. `same` becomes "covers the same core functionality" and `enhancement` asks the author to contribute the new features to the existing extension, as reviewers do by hand. Retries cover network errors, `429`, `529` and server errors. If TypeSafe still fails, the action falls back to `lexical` and adds a warning diff --git a/extension-duplicate-check/__tests__/comment.test.ts b/extension-duplicate-check/__tests__/comment.test.ts index 707958a..8e086a6 100644 --- a/extension-duplicate-check/__tests__/comment.test.ts +++ b/extension-duplicate-check/__tests__/comment.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { COMMENT_MARKER, isDuplicate, renderComment, upsertComment, type ExtensionReport } from "../src/comment"; +import { COMMENT_MARKER, renderMatchLog, isDuplicate, renderComment, upsertComment, type ExtensionReport } from "../src/comment"; import { LexicalScorer } from "../src/scorers/lexical"; import { manifestFixture, storeFixture } from "./helpers"; @@ -82,6 +82,22 @@ describe("renderComment", () => { }); }); +describe("renderMatchLog", () => { + it("logs signals as one JSON line that can't start a workflow command", async () => { + const { extension, store } = await report("spotify-controls"); + const match = { ...store[0]!, candidate: { ...store[0]!.candidate, name: "::error::\nhi" }, relation: "same" as const }; + const line = renderMatchLog(extension, match, "jev"); + expect(line.startsWith("Match: {")).toBe(true); + expect(line).not.toContain("\n"); + expect(JSON.parse(line.slice("Match: ".length))).toMatchObject({ + extension: extension.name, + scorer: "jev", + relation: "same", + signals: match.signals, + }); + }); +}); + describe("isDuplicate", () => { it("compares the best score with the threshold", async () => { const spotify = await report("spotify-controls"); diff --git a/extension-duplicate-check/action.yml b/extension-duplicate-check/action.yml index 2bd0aaa..e814db9 100644 --- a/extension-duplicate-check/action.yml +++ b/extension-duplicate-check/action.yml @@ -35,7 +35,7 @@ inputs: threshold: description: Score (0-1) at or above which a match counts as a possible duplicate. required: false - default: "0.5" + default: "0.25" min-score: description: Matches scoring below this value (0-1) are left out of the comment and the `matches` output. required: false diff --git a/extension-duplicate-check/dist/index.js b/extension-duplicate-check/dist/index.js index 0d8a1dc..aa4a69e 100644 --- a/extension-duplicate-check/dist/index.js +++ b/extension-duplicate-check/dist/index.js @@ -24655,6 +24655,10 @@ function renderComment(reports, options) { ); return lines.join("\n"); } +function renderMatchLog(extension, match, scorer) { + const { candidate, score, relation, signals } = match; + return `Match: ${JSON.stringify({ extension: extension.name, candidate: candidate.name, source: candidate.source, scorer, score, relation, signals })}`; +} function isDuplicate(report, threshold) { return [...report.store, ...report.pullRequests].some((match) => match.score >= threshold); } @@ -25532,6 +25536,7 @@ function mergeEntries(a, b) { } // src/main.ts +var LOGGED_MATCHES = 10; async function run() { try { await check(readInputs()); @@ -25556,7 +25561,7 @@ function readInputs() { scorer: getInput("scorer") || "lexical", typesafeApiKey, typesafeModel: getInput("typesafe-model") || "jev-1.14.0", - threshold: numberInput("threshold", 0.5), + threshold: numberInput("threshold", 0.25), minScore: numberInput("min-score", 0.2), topN: Math.max(1, Math.floor(numberInput("top-n", 5))), label: getInput("label"), @@ -25614,6 +25619,11 @@ async function check(inputs) { scorer = createScorer("lexical"); results = await scorer.score(added, candidates); } + added.forEach((extension, index) => { + for (const match of (results[index] ?? []).slice(0, LOGGED_MATCHES)) { + info(renderMatchLog(extension, match, scorer.name)); + } + }); const reports = added.map((extension, index) => { const matches = (results[index] ?? []).filter((match) => match.score >= inputs.minScore); return { diff --git a/extension-duplicate-check/src/comment.ts b/extension-duplicate-check/src/comment.ts index 71dfad2..b22b5f0 100644 --- a/extension-duplicate-check/src/comment.ts +++ b/extension-duplicate-check/src/comment.ts @@ -91,6 +91,12 @@ export function renderComment(reports: ExtensionReport[], options: CommentOption return lines.join("\n"); } +/** One log line per match. JSON keeps manifest text on one line, and the prefix stops it reading as a workflow command. */ +export function renderMatchLog(extension: Extension, match: Match, scorer: string): string { + const { candidate, score, relation, signals } = match; + return `Match: ${JSON.stringify({ extension: extension.name, candidate: candidate.name, source: candidate.source, scorer, score, relation, signals })}`; +} + export function isDuplicate(report: ExtensionReport, threshold: number): boolean { return [...report.store, ...report.pullRequests].some((match) => match.score >= threshold); } diff --git a/extension-duplicate-check/src/main.ts b/extension-duplicate-check/src/main.ts index 84a123b..b4b00ae 100644 --- a/extension-duplicate-check/src/main.ts +++ b/extension-duplicate-check/src/main.ts @@ -1,6 +1,6 @@ import * as core from "@actions/core"; import { context, getOctokit } from "@actions/github"; -import { isDuplicate, renderComment, upsertComment, type ExtensionReport } from "./comment"; +import { isDuplicate, renderComment, renderMatchLog, upsertComment, type ExtensionReport } from "./comment"; import { createScorer } from "./scorers"; import { fetchAddedExtensions, @@ -29,6 +29,9 @@ interface Inputs { pullRequestNumber?: number; } +/** The size of the `jev` shortlist, so every judged match reaches the run log. */ +const LOGGED_MATCHES = 10; + export async function run(): Promise { try { await check(readInputs()); @@ -54,7 +57,7 @@ function readInputs(): Inputs { scorer: core.getInput("scorer") || "lexical", typesafeApiKey, typesafeModel: core.getInput("typesafe-model") || "jev-1.14.0", - threshold: numberInput("threshold", 0.5), + threshold: numberInput("threshold", 0.25), minScore: numberInput("min-score", 0.2), topN: Math.max(1, Math.floor(numberInput("top-n", 5))), label: core.getInput("label"), @@ -120,6 +123,11 @@ async function check(inputs: Inputs): Promise { scorer = createScorer("lexical"); results = await scorer.score(added, candidates); } + added.forEach((extension, index) => { + for (const match of (results[index] ?? []).slice(0, LOGGED_MATCHES)) { + core.info(renderMatchLog(extension, match, scorer.name)); + } + }); const reports: ExtensionReport[] = added.map((extension, index) => { const matches = (results[index] ?? []).filter((match) => match.score >= inputs.minScore); return {