feat(sleep): make transcript sources and backends pluggable with entry points and public harvest helpers

CHANGELOG.md

@@ -7,6 +7,14 @@ All notable changes to SkillOpt are documented here. This project adheres to
 ## [Unreleased]
 
 ### Added
+- **Pluggable transcript sources, backends, and public harvest helpers** (#314):
+  support discovering external transcript harvesters via the
+  `skillopt_sleep.sources` entry-point group or in-memory `register_source()`,
+  and external replay/optimizer backends via `skillopt_sleep.backends` or
+  `register_backend()`. Add `--source-module pkg.mod:func` and `--backend-module
+  pkg.mod:Class` flags for standalone adapters. Expose standard harvest helpers
+  (`is_headless_replay`, `is_agent_session`, `is_meta_prompt`, `detect_feedback`,
+  `redact_secrets`) in `skillopt_sleep.harvest` for external harvesters.
 - **OfficeQA Parallel Search MCP** as an opt-in `env.search_mode=parallel`
   provider for `openai_chat`/`qwen_chat`. The default stays `offline` and no
   config selects Parallel automatically. Selecting it sends the generated

skillopt_sleep/__main__.py

@@ -100,8 +100,9 @@ def _add_common(p: argparse.ArgumentParser) -> None:
     p.add_argument("--project", default="")
     p.add_argument("--scope", default="", choices=["", "all", "invoked"])
     p.add_argument("--backend", default="",
-                   choices=["", "mock", "claude", "codex", "copilot", "cursor", "pi",
-                            "opencode", "handoff", "azure_openai"])
+                   help="replay and reflection backend (built-in, registered, or entry point)")
+    p.add_argument("--backend-module", default="",
+                   help="custom backend class or factory as 'pkg.mod:Class'")
     p.add_argument("--model", default="")
     p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary")
     p.add_argument("--cursor-path", default="", help="path to the Cursor Agent CLI")
@@ -117,8 +118,9 @@ def _add_common(p: argparse.ArgumentParser) -> None:
     p.add_argument("--cursor-home", default="", help="override ~/.cursor for Cursor session harvest")
     p.add_argument("--pi-home", default="", help="override ~/.pi for Pi session harvest")
     p.add_argument("--source", default="",
-                   choices=["", "claude", "codex", "copilot", "copilot_cli", "cursor", "pi", "opencode", "auto"],
-                   help="session transcript source")
+                   help="session transcript source (built-in, registered, or entry point)")
+    p.add_argument("--source-module", default="",
+                   help="custom transcript source as 'pkg.mod:func'")
     p.add_argument("--vscode-workspace-storage", default="",
                    help="override VS Code User/workspaceStorage root for copilot source")
     p.add_argument("--copilot-cli-session-store", default="",
@@ -160,6 +162,8 @@ def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any:
         overrides["projects"] = args.scope
     if args.backend:
         overrides["backend"] = args.backend
+    if getattr(args, "backend_module", ""):
+        overrides["backend_module"] = args.backend_module
     if args.model:
         overrides["model"] = args.model
     if getattr(args, "codex_path", ""):
@@ -182,6 +186,8 @@ def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any:
         overrides["cursor_home"] = os.path.abspath(os.path.expanduser(args.cursor_home))
     if getattr(args, "source", ""):
         overrides["transcript_source"] = args.source
+    if getattr(args, "source_module", ""):
+        overrides["source_module"] = args.source_module
     if getattr(args, "vscode_workspace_storage", ""):
         overrides["vscode_workspace_storage"] = os.path.abspath(
             os.path.expanduser(args.vscode_workspace_storage)

skillopt_sleep/backend.py

@@ -2706,6 +2706,70 @@ def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str
         return ""
 
 
+_CUSTOM_BACKENDS: Dict[str, Any] = {}
+
+
+def register_backend(name: str, factory: Any) -> None:
+    """Register a custom backend in-memory (#314)."""
+    key = (name or "").strip().lower()
+    if not key:
+        raise ValueError("Backend name cannot be blank")
+    _CUSTOM_BACKENDS[key] = factory
+
+
+def unregister_backend(name: str) -> None:
+    """Unregister a custom backend."""
+    _CUSTOM_BACKENDS.pop((name or "").strip().lower(), None)
+
+
+def load_entry_point_backends() -> Dict[str, Any]:
+    """Discover backends registered via entry points (#314)."""
+    try:
+        from importlib.metadata import entry_points
+    except ImportError:
+        return {}
+    try:
+        eps = entry_points(group="skillopt_sleep.backends")
+    except TypeError:
+        eps = entry_points().get("skillopt_sleep.backends", [])
+    out: Dict[str, Any] = {}
+    for ep in eps:
+        out[ep.name.strip().lower()] = ep.load
+    return out
+
+
+def get_supported_backends() -> List[str]:
+    """Return all known backend names: built-ins, in-memory registered, and entry points."""
+    builtins = [
+        "azure",
+        "azure-responses",
+        "claude",
+        "codex",
+        "copilot",
+        "cursor",
+        "handoff",
+        "mock",
+        "opencode",
+        "pi",
+    ]
+    names = set(builtins)
+    names.update(_CUSTOM_BACKENDS.keys())
+    names.update(load_entry_point_backends().keys())
+    return sorted(names)
+
+
+def _invoke_backend_factory(factory: Any, **kwargs: Any) -> Backend:
+    """Instantiate a backend factory or class with compatible kwargs."""
+    try:
+        return factory(**kwargs)
+    except TypeError:
+        model = kwargs.get("model", "")
+        try:
+            return factory(model=model)
+        except TypeError:
+            return factory()
+
+
 def get_backend(
     name: str,
     *,
@@ -2718,7 +2782,44 @@ def get_backend(
     opencode_tool_replay: bool = False,
     azure_endpoint: str = "",
     project_dir: str = "",
+    backend_module: str = "",
+    **kwargs: Any,
 ) -> Backend:
+    from skillopt_sleep.harvest_sources import load_module_callable
+
+    if backend_module:
+        factory = load_module_callable(backend_module)
+        return _invoke_backend_factory(
+            factory,
+            model=model,
+            claude_path=claude_path,
+            codex_path=codex_path,
+            pi_path=pi_path,
+            cursor_path=cursor_path,
+            opencode_path=opencode_path,
+            opencode_tool_replay=opencode_tool_replay,
+            azure_endpoint=azure_endpoint,
+            project_dir=project_dir,
+            **kwargs,
+        )
+
+    raw_name = name or "mock"
+    if isinstance(raw_name, str) and ":" in raw_name:
+        factory = load_module_callable(raw_name)
+        return _invoke_backend_factory(
+            factory,
+            model=model,
+            claude_path=claude_path,
+            codex_path=codex_path,
+            pi_path=pi_path,
+            cursor_path=cursor_path,
+            opencode_path=opencode_path,
+            opencode_tool_replay=opencode_tool_replay,
+            azure_endpoint=azure_endpoint,
+            project_dir=project_dir,
+            **kwargs,
+        )
+
     n = (name or "mock").strip().lower()
     if n in {"pi", "pi_cli", "pi_coding_agent", "pi-coding-agent"}:
         return PiCliBackend(model=model, pi_path=pi_path or "pi")
@@ -2750,19 +2851,55 @@ def get_backend(
         return HandoffBackend(model=model, handoff_dir=hdir)
     if n in {"mock", ""}:
         return MockBackend()
+
+    if n in _CUSTOM_BACKENDS:
+        return _invoke_backend_factory(
+            _CUSTOM_BACKENDS[n],
+            model=model,
+            claude_path=claude_path,
+            codex_path=codex_path,
+            pi_path=pi_path,
+            cursor_path=cursor_path,
+            opencode_path=opencode_path,
+            opencode_tool_replay=opencode_tool_replay,
+            azure_endpoint=azure_endpoint,
+            project_dir=project_dir,
+            **kwargs,
+        )
+
+    ep_backends = load_entry_point_backends()
+    if n in ep_backends:
+        factory = ep_backends[n]()
+        return _invoke_backend_factory(
+            factory,
+            model=model,
+            claude_path=claude_path,
+            codex_path=codex_path,
+            pi_path=pi_path,
+            cursor_path=cursor_path,
+            opencode_path=opencode_path,
+            opencode_tool_replay=opencode_tool_replay,
+            azure_endpoint=azure_endpoint,
+            project_dir=project_dir,
+            **kwargs,
+        )
+
+    supported = ", ".join(repr(s) for s in get_supported_backends())
     raise ValueError(
-        f"Unknown backend: {name!r}. Supported backends are: 'azure', 'azure-responses', "
-        "'claude', 'codex', 'copilot', 'cursor', 'handoff', 'mock', 'opencode', 'pi'."
+        f"Unknown backend: {name!r}. Supported backends are: {supported}."
     )
 
 
 def build_backend(
     *,
     backend: str = "mock",
+    backend_module: str = "",
     model: str = "",
     optimizer_backend: str = "",
+    optimizer_backend_module: str = "",
     optimizer_model: str = "",
     target_backend: str = "",
+    target_backend_module: str = "",
     target_model: str = "",
     codex_path: str = "",
     pi_path: str = "",
@@ -2780,10 +2917,18 @@ def build_backend(
     from (backend, model). ``preferences`` (free text) is attached so reflect
     uses it as a prior (set on the optimizer for dual backends).
     """
-    has_split = any([optimizer_backend, optimizer_model, target_backend, target_model])
+    has_split = any([
+        optimizer_backend,
+        optimizer_backend_module,
+        optimizer_model,
+        target_backend,
+        target_backend_module,
+        target_model,
+    ])
     if not has_split:
         be = get_backend(
             backend,
+            backend_module=backend_module,
             model=model,
             codex_path=codex_path,
             pi_path=pi_path,
@@ -2795,16 +2940,30 @@ def build_backend(
         )
         be.preferences = preferences
         return be
-    tgt = get_backend(target_backend or backend, model=target_model or model,
-                      codex_path=codex_path, pi_path=pi_path, cursor_path=cursor_path,
-                      opencode_path=opencode_path, opencode_tool_replay=opencode_tool_replay,
-                      azure_endpoint=azure_endpoint,
-                      project_dir=project_dir)
-    opt = get_backend(optimizer_backend or backend, model=optimizer_model or model,
-                      codex_path=codex_path, pi_path=pi_path, cursor_path=cursor_path,
-                      opencode_path=opencode_path, opencode_tool_replay=opencode_tool_replay,
-                      azure_endpoint=azure_endpoint,
-                      project_dir=project_dir)
+    tgt = get_backend(
+        target_backend or backend,
+        backend_module=target_backend_module or backend_module,
+        model=target_model or model,
+        codex_path=codex_path,
+        pi_path=pi_path,
+        cursor_path=cursor_path,
+        opencode_path=opencode_path,
+        opencode_tool_replay=opencode_tool_replay,
+        azure_endpoint=azure_endpoint,
+        project_dir=project_dir,
+    )
+    opt = get_backend(
+        optimizer_backend or backend,
+        backend_module=optimizer_backend_module or backend_module,
+        model=optimizer_model or model,
+        codex_path=codex_path,
+        pi_path=pi_path,
+        cursor_path=cursor_path,
+        opencode_path=opencode_path,
+        opencode_tool_replay=opencode_tool_replay,
+        azure_endpoint=azure_endpoint,
+        project_dir=project_dir,
+    )
     opt.preferences = preferences  # reflect runs on the optimizer
     dual = DualBackend(target=tgt, optimizer=opt)
     dual.preferences = preferences

skillopt_sleep/cycle.py

@@ -32,7 +32,6 @@
 from skillopt_sleep.harvest_sources import harvest_for_config, normalize_transcript_source
 from skillopt_sleep.memory import ensure_skill_scaffold
 from skillopt_sleep.mine import group_tasks_by_skill_hint, mine
-from skillopt_sleep.replay import aggregate_scores, replay_batch
 from skillopt_sleep.multi_skill import (
     BUDGET_REACHED,
     SKIPPED,
@@ -42,6 +41,7 @@
     consolidate_groups,
     skill_group_reports,
 )
+from skillopt_sleep.replay import aggregate_scores, replay_batch
 from skillopt_sleep.skill_resolver import resolve_skill, skill_search_roots
 from skillopt_sleep.staging import (
     SkillProposal,
@@ -684,10 +684,13 @@ def run_sleep_cycle(
 
     backend = backend or build_backend(
         backend=cfg.get("backend", "mock"),
+        backend_module=cfg.get("backend_module", ""),
         model=cfg.get("model", ""),
         optimizer_backend=cfg.get("optimizer_backend", ""),
+        optimizer_backend_module=cfg.get("optimizer_backend_module", ""),
         optimizer_model=cfg.get("optimizer_model", ""),
         target_backend=cfg.get("target_backend", ""),
+        target_backend_module=cfg.get("target_backend_module", ""),
         target_model=cfg.get("target_model", ""),
         codex_path=cfg.get("codex_path", ""),
         pi_path=cfg.get("pi_path", ""),

skillopt_sleep/harvest.py

@@ -23,7 +23,6 @@
 
 from skillopt_sleep.types import SessionDigest
 
-
 # Heuristic phrases that signal the user (dis)approving of prior output.
 # English-only by default. Users whose sessions are in another language can add
 # their own phrases via the SKILLOPT_SLEEP_NEG_FEEDBACK / _POS_FEEDBACK env vars
@@ -689,3 +688,23 @@ def _eligible(d: SessionDigest) -> bool:
             sessions=len(digests),
         )
     return digests
+
+
+# ── Public helper API for built-in and out-of-tree harvesters (#314) ──────────
+from skillopt_sleep.staging import redact_secrets  # noqa: E402
+
+is_headless_replay = _is_headless_replay
+is_agent_session = _is_agent_session
+is_meta_prompt = _is_meta_prompt
+detect_feedback = _detect_feedback
+
+__all__ = [
+    "harvest",
+    "format_harvest_note",
+    "is_headless_replay",
+    "is_agent_session",
+    "is_meta_prompt",
+    "detect_feedback",
+    "redact_secrets",
+]
+