From 72ba36d88842633405e29586eb0d63f9ed61c5c4 Mon Sep 17 00:00:00 2001 From: Dhruv Suthar Date: Mon, 2 Nov 2026 18:11:51 +0530 Subject: [PATCH] extension-duplicate-check: Log match signals and lower threshold to 0.25 Each run now logs up to 10 matches per extension as JSON, including covers and services, so results can be reviewed from run logs. The lower threshold flags matches like Everything Search, which scored 0.40 against the Store's Everything. --- extension-duplicate-check/README.md | 10 ++++++++-- .../__tests__/comment.test.ts | 18 +++++++++++++++++- extension-duplicate-check/action.yml | 2 +- extension-duplicate-check/dist/index.js | 12 +++++++++++- extension-duplicate-check/src/comment.ts | 6 ++++++ extension-duplicate-check/src/main.ts | 12 ++++++++++-- 6 files changed, 53 insertions(+), 7 deletions(-) diff --git a/extension-duplicate-check/README.md b/extension-duplicate-check/README.md index f5557ab..9ca9244 100644 --- a/extension-duplicate-check/README.md +++ b/extension-duplicate-check/README.md @@ -74,7 +74,7 @@ raycast/extensions doesn't run Dependabot, so bump the SHA by hand after changin | `scorer` | `lexical` | `lexical` scores in process. `jev` shortlists with `lexical` and asks TypeSafe's Jev model to judge each match. | | `typesafe-api-key` | | TypeSafe API key, required by the `jev` scorer. Pass it from a secret. | | `typesafe-model` | `jev-1.14.0` | TypeSafe model name or alias for the `jev` scorer. Pinned because thresholds are tuned per version and `jev-latest` can move. | -| `threshold` | `0.5` | Score (0–1) at or above which a match counts as a possible duplicate. | +| `threshold` | `0.25` | Score (0–1) at or above which a match counts as a possible duplicate. | | `min-score` | `0.2` | Matches below this score are left out of the comment and the `matches` output. | | `top-n` | `5` | Number of matches listed per source (store and open pull requests) for each new extension. | | `label` | | Label added when a possible duplicate is found and removed otherwise. Empty disables labeling. | @@ -131,7 +131,7 @@ The `lexical` scorer combines two signals: 0.8 count as unrelated. The score is `1 - (1 - content) × (1 - 0.5 × name)`, so either signal alone can raise it and identical names alone reach -0.5. The default threshold of 0.5 comes from a backtest over three months of new-extension pull requests: it flagged +0.5. A threshold of 0.5 in a backtest over three months of new-extension pull requests flagged 37% of those closed as duplicates of a store extension and 11% of those that were merged. The lexical scorer misses duplicates that describe the same thing in different words. In the same backtest it ranked @@ -168,6 +168,12 @@ over the same three months, with `jev-1.14.0`: Most of the merged extensions it flags overlap a lot with an existing one, such as WHost and iHosts, or Ghq Plus and ghq, so they are worth a reviewer's look anyway. Of the 17 duplicates it misses at 0.5, 12 score below 0.4. +The default threshold is 0.25, lower than anything in this backtest, so that matches like Everything Search, which +scored 0.40 against the Store's Everything, get flagged. Expect more flagged merged extensions than the table shows. + +Each run logs up to 10 matches per new extension as `Match: {...}` JSON lines, with `covers`, `services` and the other +signals, including matches below `min-score`. + The relation also words the comment. `same` becomes "covers the same core functionality" and `enhancement` asks the author to contribute the new features to the existing extension, as reviewers do by hand. Retries cover network errors, `429`, `529` and server errors. If TypeSafe still fails, the action falls back to `lexical` and adds a warning diff --git a/extension-duplicate-check/__tests__/comment.test.ts b/extension-duplicate-check/__tests__/comment.test.ts index 707958a..8e086a6 100644 --- a/extension-duplicate-check/__tests__/comment.test.ts +++ b/extension-duplicate-check/__tests__/comment.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { COMMENT_MARKER, isDuplicate, renderComment, upsertComment, type ExtensionReport } from "../src/comment"; +import { COMMENT_MARKER, renderMatchLog, isDuplicate, renderComment, upsertComment, type ExtensionReport } from "../src/comment"; import { LexicalScorer } from "../src/scorers/lexical"; import { manifestFixture, storeFixture } from "./helpers"; @@ -82,6 +82,22 @@ describe("renderComment", () => { }); }); +describe("renderMatchLog", () => { + it("logs signals as one JSON line that can't start a workflow command", async () => { + const { extension, store } = await report("spotify-controls"); + const match = { ...store[0]!, candidate: { ...store[0]!.candidate, name: "::error::\nhi" }, relation: "same" as const }; + const line = renderMatchLog(extension, match, "jev"); + expect(line.startsWith("Match: {")).toBe(true); + expect(line).not.toContain("\n"); + expect(JSON.parse(line.slice("Match: ".length))).toMatchObject({ + extension: extension.name, + scorer: "jev", + relation: "same", + signals: match.signals, + }); + }); +}); + describe("isDuplicate", () => { it("compares the best score with the threshold", async () => { const spotify = await report("spotify-controls"); diff --git a/extension-duplicate-check/action.yml b/extension-duplicate-check/action.yml index 2bd0aaa..e814db9 100644 --- a/extension-duplicate-check/action.yml +++ b/extension-duplicate-check/action.yml @@ -35,7 +35,7 @@ inputs: threshold: description: Score (0-1) at or above which a match counts as a possible duplicate. required: false - default: "0.5" + default: "0.25" min-score: description: Matches scoring below this value (0-1) are left out of the comment and the `matches` output. required: false diff --git a/extension-duplicate-check/dist/index.js b/extension-duplicate-check/dist/index.js index 0d8a1dc..aa4a69e 100644 --- a/extension-duplicate-check/dist/index.js +++ b/extension-duplicate-check/dist/index.js @@ -24655,6 +24655,10 @@ function renderComment(reports, options) { ); return lines.join("\n"); } +function renderMatchLog(extension, match, scorer) { + const { candidate, score, relation, signals } = match; + return `Match: ${JSON.stringify({ extension: extension.name, candidate: candidate.name, source: candidate.source, scorer, score, relation, signals })}`; +} function isDuplicate(report, threshold) { return [...report.store, ...report.pullRequests].some((match) => match.score >= threshold); } @@ -25532,6 +25536,7 @@ function mergeEntries(a, b) { } // src/main.ts +var LOGGED_MATCHES = 10; async function run() { try { await check(readInputs()); @@ -25556,7 +25561,7 @@ function readInputs() { scorer: getInput("scorer") || "lexical", typesafeApiKey, typesafeModel: getInput("typesafe-model") || "jev-1.14.0", - threshold: numberInput("threshold", 0.5), + threshold: numberInput("threshold", 0.25), minScore: numberInput("min-score", 0.2), topN: Math.max(1, Math.floor(numberInput("top-n", 5))), label: getInput("label"), @@ -25614,6 +25619,11 @@ async function check(inputs) { scorer = createScorer("lexical"); results = await scorer.score(added, candidates); } + added.forEach((extension, index) => { + for (const match of (results[index] ?? []).slice(0, LOGGED_MATCHES)) { + info(renderMatchLog(extension, match, scorer.name)); + } + }); const reports = added.map((extension, index) => { const matches = (results[index] ?? []).filter((match) => match.score >= inputs.minScore); return { diff --git a/extension-duplicate-check/src/comment.ts b/extension-duplicate-check/src/comment.ts index 71dfad2..b22b5f0 100644 --- a/extension-duplicate-check/src/comment.ts +++ b/extension-duplicate-check/src/comment.ts @@ -91,6 +91,12 @@ export function renderComment(reports: ExtensionReport[], options: CommentOption return lines.join("\n"); } +/** One log line per match. JSON keeps manifest text on one line, and the prefix stops it reading as a workflow command. */ +export function renderMatchLog(extension: Extension, match: Match, scorer: string): string { + const { candidate, score, relation, signals } = match; + return `Match: ${JSON.stringify({ extension: extension.name, candidate: candidate.name, source: candidate.source, scorer, score, relation, signals })}`; +} + export function isDuplicate(report: ExtensionReport, threshold: number): boolean { return [...report.store, ...report.pullRequests].some((match) => match.score >= threshold); } diff --git a/extension-duplicate-check/src/main.ts b/extension-duplicate-check/src/main.ts index 84a123b..b4b00ae 100644 --- a/extension-duplicate-check/src/main.ts +++ b/extension-duplicate-check/src/main.ts @@ -1,6 +1,6 @@ import * as core from "@actions/core"; import { context, getOctokit } from "@actions/github"; -import { isDuplicate, renderComment, upsertComment, type ExtensionReport } from "./comment"; +import { isDuplicate, renderComment, renderMatchLog, upsertComment, type ExtensionReport } from "./comment"; import { createScorer } from "./scorers"; import { fetchAddedExtensions, @@ -29,6 +29,9 @@ interface Inputs { pullRequestNumber?: number; } +/** The size of the `jev` shortlist, so every judged match reaches the run log. */ +const LOGGED_MATCHES = 10; + export async function run(): Promise { try { await check(readInputs()); @@ -54,7 +57,7 @@ function readInputs(): Inputs { scorer: core.getInput("scorer") || "lexical", typesafeApiKey, typesafeModel: core.getInput("typesafe-model") || "jev-1.14.0", - threshold: numberInput("threshold", 0.5), + threshold: numberInput("threshold", 0.25), minScore: numberInput("min-score", 0.2), topN: Math.max(1, Math.floor(numberInput("top-n", 5))), label: core.getInput("label"), @@ -120,6 +123,11 @@ async function check(inputs: Inputs): Promise { scorer = createScorer("lexical"); results = await scorer.score(added, candidates); } + added.forEach((extension, index) => { + for (const match of (results[index] ?? []).slice(0, LOGGED_MATCHES)) { + core.info(renderMatchLog(extension, match, scorer.name)); + } + }); const reports: ExtensionReport[] = added.map((extension, index) => { const matches = (results[index] ?? []).filter((match) => match.score >= inputs.minScore); return {