extension-duplicate-check: Log match signals and lower threshold to 0.25

extension-duplicate-check/README.md

@@ -74,7 +74,7 @@ raycast/extensions doesn't run Dependabot, so bump the SHA by hand after changin
 | `scorer`                   | `lexical`                    | `lexical` scores in process. `jev` shortlists with `lexical` and asks TypeSafe's Jev model to judge each match.                                                                                          |
 | `typesafe-api-key`         |                              | TypeSafe API key, required by the `jev` scorer. Pass it from a secret.                                                                                                                                   |
 | `typesafe-model`           | `jev-1.14.0`                 | TypeSafe model name or alias for the `jev` scorer. Pinned because thresholds are tuned per version and `jev-latest` can move.                                                                            |
-| `threshold`                | `0.5`                        | Score (0–1) at or above which a match counts as a possible duplicate.                                                                                                                                    |
+| `threshold`                | `0.25`                       | Score (0–1) at or above which a match counts as a possible duplicate.                                                                                                                                    |
 | `min-score`                | `0.2`                        | Matches below this score are left out of the comment and the `matches` output.                                                                                                                           |
 | `top-n`                    | `5`                          | Number of matches listed per source (store and open pull requests) for each new extension.                                                                                                               |
 | `label`                    |                              | Label added when a possible duplicate is found and removed otherwise. Empty disables labeling.                                                                                                           |
@@ -131,7 +131,7 @@ The `lexical` scorer combines two signals:
   0.8 count as unrelated.
 
 The score is `1 - (1 - content) × (1 - 0.5 × name)`, so either signal alone can raise it and identical names alone reach
-0.5. The default threshold of 0.5 comes from a backtest over three months of new-extension pull requests: it flagged
+0.5. A threshold of 0.5 in a backtest over three months of new-extension pull requests flagged
 37% of those closed as duplicates of a store extension and 11% of those that were merged.
 
 The lexical scorer misses duplicates that describe the same thing in different words. In the same backtest it ranked
@@ -168,6 +168,12 @@ over the same three months, with `jev-1.14.0`:
 Most of the merged extensions it flags overlap a lot with an existing one, such as WHost and iHosts, or Ghq Plus and
 ghq, so they are worth a reviewer's look anyway. Of the 17 duplicates it misses at 0.5, 12 score below 0.4.
 
+The default threshold is 0.25, lower than anything in this backtest, so that matches like Everything Search, which
+scored 0.40 against the Store's Everything, get flagged. Expect more flagged merged extensions than the table shows.
+
+Each run logs up to 10 matches per new extension as `Match: {...}` JSON lines, with `covers`, `services` and the other
+signals, including matches below `min-score`.
+
 The relation also words the comment. `same` becomes "covers the same core functionality" and `enhancement` asks the
 author to contribute the new features to the existing extension, as reviewers do by hand. Retries cover network
 errors, `429`, `529` and server errors. If TypeSafe still fails, the action falls back to `lexical` and adds a warning

extension-duplicate-check/__tests__/comment.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { COMMENT_MARKER, isDuplicate, renderComment, upsertComment, type ExtensionReport } from "../src/comment";
+import { COMMENT_MARKER, renderMatchLog, isDuplicate, renderComment, upsertComment, type ExtensionReport } from "../src/comment";
 import { LexicalScorer } from "../src/scorers/lexical";
 import { manifestFixture, storeFixture } from "./helpers";
 
@@ -82,6 +82,22 @@ describe("renderComment", () => {
   });
 });
 
+describe("renderMatchLog", () => {
+  it("logs signals as one JSON line that can't start a workflow command", async () => {
+    const { extension, store } = await report("spotify-controls");
+    const match = { ...store[0]!, candidate: { ...store[0]!.candidate, name: "::error::\nhi" }, relation: "same" as const };
+    const line = renderMatchLog(extension, match, "jev");
+    expect(line.startsWith("Match: {")).toBe(true);
+    expect(line).not.toContain("\n");
+    expect(JSON.parse(line.slice("Match: ".length))).toMatchObject({
+      extension: extension.name,
+      scorer: "jev",
+      relation: "same",
+      signals: match.signals,
+    });
+  });
+});
+
 describe("isDuplicate", () => {
   it("compares the best score with the threshold", async () => {
     const spotify = await report("spotify-controls");

extension-duplicate-check/action.yml

@@ -35,7 +35,7 @@ inputs:
   threshold:
     description: Score (0-1) at or above which a match counts as a possible duplicate.
     required: false
-    default: "0.5"
+    default: "0.25"
   min-score:
     description: Matches scoring below this value (0-1) are left out of the comment and the `matches` output.
     required: false

extension-duplicate-check/dist/index.js

@@ -24655,6 +24655,10 @@ function renderComment(reports, options) {
   );
   return lines.join("\n");
 }
+function renderMatchLog(extension, match, scorer) {
+  const { candidate, score, relation, signals } = match;
+  return `Match: ${JSON.stringify({ extension: extension.name, candidate: candidate.name, source: candidate.source, scorer, score, relation, signals })}`;
+}
 function isDuplicate(report, threshold) {
   return [...report.store, ...report.pullRequests].some((match) => match.score >= threshold);
 }
@@ -25532,6 +25536,7 @@ function mergeEntries(a, b) {
 }
 
 // src/main.ts
+var LOGGED_MATCHES = 10;
 async function run() {
   try {
     await check(readInputs());
@@ -25556,7 +25561,7 @@ function readInputs() {
     scorer: getInput("scorer") || "lexical",
     typesafeApiKey,
     typesafeModel: getInput("typesafe-model") || "jev-1.14.0",
-    threshold: numberInput("threshold", 0.5),
+    threshold: numberInput("threshold", 0.25),
     minScore: numberInput("min-score", 0.2),
     topN: Math.max(1, Math.floor(numberInput("top-n", 5))),
     label: getInput("label"),
@@ -25614,6 +25619,11 @@ async function check(inputs) {
     scorer = createScorer("lexical");
     results = await scorer.score(added, candidates);
   }
+  added.forEach((extension, index) => {
+    for (const match of (results[index] ?? []).slice(0, LOGGED_MATCHES)) {
+      info(renderMatchLog(extension, match, scorer.name));
+    }
+  });
   const reports = added.map((extension, index) => {
     const matches = (results[index] ?? []).filter((match) => match.score >= inputs.minScore);
     return {

extension-duplicate-check/src/comment.ts

@@ -91,6 +91,12 @@ export function renderComment(reports: ExtensionReport[], options: CommentOption
   return lines.join("\n");
 }
 
+/** One log line per match. JSON keeps manifest text on one line, and the prefix stops it reading as a workflow command. */
+export function renderMatchLog(extension: Extension, match: Match, scorer: string): string {
+  const { candidate, score, relation, signals } = match;
+  return `Match: ${JSON.stringify({ extension: extension.name, candidate: candidate.name, source: candidate.source, scorer, score, relation, signals })}`;
+}
+
 export function isDuplicate(report: ExtensionReport, threshold: number): boolean {
   return [...report.store, ...report.pullRequests].some((match) => match.score >= threshold);
 }

extension-duplicate-check/src/main.ts

@@ -1,6 +1,6 @@
 import * as core from "@actions/core";
 import { context, getOctokit } from "@actions/github";
-import { isDuplicate, renderComment, upsertComment, type ExtensionReport } from "./comment";
+import { isDuplicate, renderComment, renderMatchLog, upsertComment, type ExtensionReport } from "./comment";
 import { createScorer } from "./scorers";
 import {
   fetchAddedExtensions,
@@ -29,6 +29,9 @@ interface Inputs {
   pullRequestNumber?: number;
 }
 
+/** The size of the `jev` shortlist, so every judged match reaches the run log. */
+const LOGGED_MATCHES = 10;
+
 export async function run(): Promise<void> {
   try {
     await check(readInputs());
@@ -54,7 +57,7 @@ function readInputs(): Inputs {
     scorer: core.getInput("scorer") || "lexical",
     typesafeApiKey,
     typesafeModel: core.getInput("typesafe-model") || "jev-1.14.0",
-    threshold: numberInput("threshold", 0.5),
+    threshold: numberInput("threshold", 0.25),
     minScore: numberInput("min-score", 0.2),
     topN: Math.max(1, Math.floor(numberInput("top-n", 5))),
     label: core.getInput("label"),
@@ -120,6 +123,11 @@ async function check(inputs: Inputs): Promise<void> {
     scorer = createScorer("lexical");
     results = await scorer.score(added, candidates);
   }
+  added.forEach((extension, index) => {
+    for (const match of (results[index] ?? []).slice(0, LOGGED_MATCHES)) {
+      core.info(renderMatchLog(extension, match, scorer.name));
+    }
+  });
   const reports: ExtensionReport[] = added.map((extension, index) => {
     const matches = (results[index] ?? []).filter((match) => match.score >= inputs.minScore);
     return {